From 3fcb10fc7041d75dfe320043e208680e303f567d Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 12:45:07 -0700 Subject: [PATCH 001/223] =?UTF-8?q?=F0=9F=90=9B=20agents:=20Copilot=20CLI?= =?UTF-8?q?=20ET=5FEXEC=20binary=20fails=20with=20ENOEXEC=20on=20kernel=20?= =?UTF-8?q?6.12=20=E2=80=94=20exec=20via=20ld.so=20+=20startup=20preflight?= =?UTF-8?q?=20(#17737)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #17729. Refs #17642. Signed-off-by: Andrew Anderson --- bin/install-copilot-exec-shim.sh | 78 ++++++++ .../fixed-17729-copilot-enoexec-ldso-shim.md | 1 + src/Dockerfile | 8 + src/Dockerfile.contributor | 4 + src/crux/agent/exec_format.go | 178 ++++++++++++++++++ src/crux/agent/exec_format_test.go | 87 +++++++++ src/crux/agent/manager_credentials.go | 20 ++ src/crux/runtime/runtime.go | 24 +++ 8 files changed, 400 insertions(+) create mode 100755 bin/install-copilot-exec-shim.sh create mode 100644 changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md create mode 100644 src/crux/agent/exec_format.go create mode 100644 src/crux/agent/exec_format_test.go diff --git a/bin/install-copilot-exec-shim.sh b/bin/install-copilot-exec-shim.sh new file mode 100755 index 0000000000..1db16d94c7 --- /dev/null +++ b/bin/install-copilot-exec-shim.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# install-copilot-exec-shim.sh — wrap the Copilot CLI platform binary so it is +# started through the glibc dynamic loader instead of execve'd directly. +# +# Why (hivecommons/hive#17729): the @github/copilot platform package ships a +# Bun single-file executable — an ELF64 ET_EXEC with a 0x1000-aligned appended +# JS segment after 0x10000-aligned code/data LOADs. Linux 6.12 (seen on OKE +# UEK8 6.12.0-207, aarch64) rejects that layout in binfmt_elf with ENOEXEC +# ("Exec format error"); the npm-loader shim's sh fallback then "runs" the +# ELF as a script and every copilot agent crash-loops. glibc's ld.so maps the +# very same file without complaint, so `ld.so ` is a drop-in. +# +# Build-time only. Idempotent. Tolerant of a missing copilot install (the +# Dockerfile layer that installs it is itself tolerant). Usage: +# install-copilot-exec-shim.sh [npm-global-root] +set -eu + +ROOT="${1:-$(npm root -g 2>/dev/null || echo /usr/local/lib/node_modules)}" +PKG_DIR="$ROOT/@github/copilot/node_modules/@github" + +# Arch → loader, resolved at BUILD time for the target arch. The wrapper falls +# back to a direct exec when the loader is absent (musl, unusual distros). +case "$(uname -m)" in + aarch64|arm64) LDSO=/lib/ld-linux-aarch64.so.1 ;; + x86_64|amd64) LDSO=/lib64/ld-linux-x86-64.so.2 ;; + *) LDSO="" ;; +esac + +if [ ! -d "$PKG_DIR" ]; then + echo "install-copilot-exec-shim: no @github/copilot platform package under $PKG_DIR — skipping" + exit 0 +fi + +installed=0 +for dir in "$PKG_DIR"/copilot-linux-*; do + [ -d "$dir" ] || continue + bin="$dir/copilot" + real="$dir/copilot.real" + if [ -f "$real" ] && head -c 2 "$bin" 2>/dev/null | grep -q '#!'; then + echo "install-copilot-exec-shim: $bin already wrapped" + installed=$((installed + 1)) + continue + fi + if [ ! -f "$bin" ]; then + echo "install-copilot-exec-shim: $bin missing — skipping" + continue + fi + mv "$bin" "$real" + cat > "$bin" </dev/null || true + chmod 0755 "$bin" "$real" + echo "install-copilot-exec-shim: wrapped $bin via ${LDSO:-direct exec}" + installed=$((installed + 1)) +done + +if [ "$installed" -eq 0 ]; then + echo "install-copilot-exec-shim: no copilot-linux-* platform binary found — skipping" + exit 0 +fi + +# Prove the wrapped CLI starts, through the npm shim, the way an agent runs it. +if ! copilot --version >/dev/null 2>&1; then + echo "install-copilot-exec-shim: WARNING: 'copilot --version' failed after wrapping" >&2 + exit 0 +fi +echo "install-copilot-exec-shim: copilot --version OK ($(copilot --version 2>/dev/null | head -1))" diff --git a/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md b/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md new file mode 100644 index 0000000000..80a3ef1606 --- /dev/null +++ b/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md @@ -0,0 +1 @@ +- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). diff --git a/src/Dockerfile b/src/Dockerfile index ac69b5335e..65c9b3a47d 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -291,6 +291,14 @@ ARG COPILOT_VERSION=1.0.88 RUN npm install -g --ignore-scripts --no-fund --no-audit \ "@github/copilot@${COPILOT_VERSION}" && npm cache clean --force || \ echo "WARN: GitHub Copilot CLI install failed — skipping" +# Wrap the platform binary so it starts through glibc's ld.so (#17729): the +# Bun single-file ET_EXEC the package ships is rejected with ENOEXEC by Linux +# 6.12 binfmt_elf (OKE UEK8 upgrade took every copilot agent down), while the +# dynamic loader maps the same file fine. The installer is idempotent and a +# no-op when the copilot layer above was skipped. +COPY bin/install-copilot-exec-shim.sh /usr/local/bin/install-copilot-exec-shim.sh +RUN chmod +x /usr/local/bin/install-copilot-exec-shim.sh && \ + /usr/local/bin/install-copilot-exec-shim.sh /usr/local/lib/node_modules # --- Layer 3b: GitHub Copilot SDK (pinned; powers SDK-based model discovery) --- # The dashboard's copilot model discovery prefers a probe through the official diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 17fa574e92..be037588fc 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -271,6 +271,10 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \ && npm cache clean --force \ && node "$(npm root -g)/@anthropic-ai/claude-code/install.cjs" \ && claude --version +# ld.so wrapper for the copilot platform binary (#17729) — see src/Dockerfile. +COPY bin/install-copilot-exec-shim.sh /usr/local/bin/install-copilot-exec-shim.sh +RUN chmod +x /usr/local/bin/install-copilot-exec-shim.sh && \ + /usr/local/bin/install-copilot-exec-shim.sh "$(npm root -g)" # Install Muse Code (backend: muse). Mirrors Layer 8.3 of src/Dockerfile so a # containerized contributor relay can actually run the muse backend, instead of diff --git a/src/crux/agent/exec_format.go b/src/crux/agent/exec_format.go new file mode 100644 index 0000000000..90a010a6a1 --- /dev/null +++ b/src/crux/agent/exec_format.go @@ -0,0 +1,178 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os/exec" + "strings" + "syscall" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// ENOEXEC detection (hivecommons/hive#17729). +// +// After the OKE nodes moved to Linux 6.12 (UEK8), the kernel's ELF loader +// rejected the Copilot CLI platform binary — a Bun single-file ET_EXEC — with +// ENOEXEC. Nothing in the launch path said so: the npm shim's sh fell back to +// interpreting the ELF as a script ("copilot: 1: : not found"), the hang +// diagnostic saw no copilot output and timed out, and the restart-storm +// breaker backed every agent off for up to 30 minutes. Fifteen hives, zero +// agents, and the only log line was "copilot hung with no output". +// +// Two things make that loud now. paneShowsExecFormatError recognises the +// kernel's message and the sh-fallback signature in a diagnostic pane so the +// diagnostic fails the agent with a remedy instead of relaunching. And +// PreflightBackendBinaries execs every installed agent CLI once at boot so the +// same breakage is one ERROR line at startup, before any agent is launched. + +// execFormatErrorIndicators are the pane signatures of an ELF the kernel +// refused: the libc strerror text, and what dash/bash print when they fall +// back to running the binary as a script (the ELF's PT_INTERP string lands on +// the first "line", so the "command" sh reports is the loader path). +var execFormatErrorIndicators = []string{ + "Exec format error", + "cannot execute binary file", + "ld-linux-aarch64.so.1GNU", + "ld-linux-x86-64.so.2GNU", +} + +// paneShowsExecFormatError reports whether a diagnostic pane shows the CLI +// binary being rejected by the kernel rather than failing on its own. +func paneShowsExecFormatError(lines []string) bool { + for _, line := range lines { + for _, ind := range execFormatErrorIndicators { + if strings.Contains(line, ind) { + return true + } + } + // sh fell back to "running" the ELF as a script and reported the + // PT_INTERP string (the first printable run in the file) as a + // missing command. + if strings.Contains(line, "ld-linux-") && strings.Contains(line, ": not found") { + return true + } + } + return false +} + +// execFormatRemedy is the operator-facing explanation attached to every +// ENOEXEC finding. It names the one thing that fixes it (run the binary +// through ld.so, which the hive image now does via +// bin/install-copilot-exec-shim.sh) so nobody has to rediscover it from a +// pane full of "not found". +const execFormatRemedy = "the kernel refused to exec the CLI binary (ENOEXEC / Exec format error): " + + "usually a Bun/Node single-file ET_EXEC on a kernel whose ELF loader rejects its segment layout (seen with Linux 6.12 on aarch64). " + + "Relaunching cannot help. Start it through the dynamic loader instead (`/lib/ld-linux-aarch64.so.1 ` / `/lib64/ld-linux-x86-64.so.2 `) " + + "or roll a hive image built with bin/install-copilot-exec-shim.sh — see hivecommons/hive#17729" + +// execFormatLastError is what the agent's LastError is set to when the +// diagnostic sees ENOEXEC. +func execFormatLastError(binary string) string { + return fmt.Sprintf("%s cannot be executed on this kernel: %s", binary, execFormatRemedy) +} + +// Preflight pacing. Vars so tests can shrink them. +var ( + binaryPreflightTimeout = 20 * time.Second +) + +// BinaryPreflightFinding is one CLI binary the preflight could not start. +type BinaryPreflightFinding struct { + Backend string + Binary string + Path string + // ExecFormat is true when the failure is the kernel refusing the ELF + // (ENOEXEC) rather than the CLI exiting non-zero on its own. + ExecFormat bool + Err error + Output string +} + +func (f BinaryPreflightFinding) String() string { + if f.ExecFormat { + return fmt.Sprintf("backend %s: %s", f.Backend, execFormatLastError(f.Path)) + } + return fmt.Sprintf("backend %s: %s --version failed: %v", f.Backend, f.Path, f.Err) +} + +// classifyPreflightError decides whether a `--version` failure is the kernel +// refusing the binary. exec.Command surfaces ENOEXEC directly from execve +// (it does not go through a shell), and some CLIs are themselves shims that +// spawn the real binary and print the kernel's message. +func classifyPreflightError(err error, output string) bool { + if errors.Is(err, syscall.ENOEXEC) { + return true + } + return paneShowsExecFormatError(strings.Split(output, "\n")) +} + +// PreflightBackendBinaries runs ` --version` once for every CLI backend +// in use (or every installed CLI backend when backends is empty), with a +// timeout per binary, and returns the ones that could not start. Binaries +// not on PATH are skipped — an absent CLI is a configuration matter the +// launch path already reports; this is about CLIs that are present and +// broken. It never blocks boot: findings are returned for the caller to log. +func PreflightBackendBinaries(ctx context.Context, backends []string) []BinaryPreflightFinding { + if len(backends) == 0 { + backends = config.CLIBackends + } + seen := map[string]bool{} + var findings []BinaryPreflightFinding + for _, backend := range backends { + binary, err := backendBinaryName(backend) + if err != nil || seen[binary] { + continue + } + seen[binary] = true + path, err := exec.LookPath(binary) + if err != nil { + continue + } + if f, bad := preflightOneBinary(ctx, backend, binary, path); bad { + findings = append(findings, f) + } + } + return findings +} + +func preflightOneBinary(ctx context.Context, backend, binary, path string) (BinaryPreflightFinding, bool) { + ctx, cancel := context.WithTimeout(ctx, binaryPreflightTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, path, "--version") + cmd.Env = append(cmd.Environ(), "NO_COLOR=1", "CI=1") + out, err := cmd.CombinedOutput() + output := string(out) + if err == nil { + return BinaryPreflightFinding{}, false + } + execFormat := classifyPreflightError(err, output) + // A CLI that runs but dislikes `--version` (exit 1, usage text) is not + // what this preflight is for; only the kernel-level refusal is a finding. + // A timeout is not either — a slow cold start is the hang diagnostic's job. + if !execFormat { + return BinaryPreflightFinding{}, false + } + return BinaryPreflightFinding{ + Backend: backend, + Binary: binary, + Path: path, + ExecFormat: true, + Err: err, + Output: truncateStr(strings.TrimSpace(output), 300), + }, true +} + +// LogBackendBinaryPreflight runs PreflightBackendBinaries and logs each +// finding at ERROR with the remedy. Called once at boot by cmd/hive. +func LogBackendBinaryPreflight(ctx context.Context, logger *slog.Logger, backends []string) []BinaryPreflightFinding { + findings := PreflightBackendBinaries(ctx, backends) + for _, f := range findings { + logger.Error("agent CLI preflight: binary cannot be executed on this kernel", + "backend", f.Backend, "binary", f.Path, "error", f.Err, "output", f.Output, "remedy", execFormatRemedy) + } + return findings +} diff --git a/src/crux/agent/exec_format_test.go b/src/crux/agent/exec_format_test.go new file mode 100644 index 0000000000..7ddd159478 --- /dev/null +++ b/src/crux/agent/exec_format_test.go @@ -0,0 +1,87 @@ +package agent + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// Pane signatures from the OKE 6.12 outage (hivecommons/hive#17729): the npm +// shim's sh fell back to running the ELF as a script, so the pane had no +// copilot output at all — only sh "not found" lines naming the PT_INTERP. +func TestPaneShowsExecFormatError(t *testing.T) { + cases := []struct { + name string + pane string + want bool + }{ + {"kernel strerror", "sh: 1: /usr/local/lib/node_modules/@github/copilot/node_modules/@github/copilot-linux-arm64/copilot: Exec format error", true}, + {"bash wording", "bash: ./copilot: cannot execute binary file: Exec format error", true}, + {"sh fallback interp line", "copilot: 1: : not found\ncopilot: 2: /lib/ld-linux-aarch64.so.1GNU: not found", true}, + {"sh fallback x86 interp", "/lib64/ld-linux-x86-64.so.2GNU: not found", true}, + {"ordinary not found is not enoexec", "sh: 1: foo: not found", false}, + {"auth failure is not enoexec", "Bad credentials\n❯", false}, + {"ready prompt", "GitHub Copilot CLI 1.0.88\n❯", false}, + {"empty", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := paneShowsExecFormatError(strings.Split(tc.pane, "\n")); got != tc.want { + t.Fatalf("paneShowsExecFormatError(%q) = %v, want %v", tc.pane, got, tc.want) + } + }) + } +} + +func TestExecFormatLastErrorNamesBinaryAndRemedy(t *testing.T) { + got := execFormatLastError("/usr/local/bin/copilot") + for _, want := range []string{"/usr/local/bin/copilot", "ENOEXEC", "ld-linux-aarch64.so.1", "#17729"} { + if !strings.Contains(got, want) { + t.Fatalf("LastError %q lacks %q", got, want) + } + } +} + +// A real execve of a non-ELF, non-shebang file is the kernel's ENOEXEC; a +// binary that merely exits non-zero is not a finding. +func TestPreflightOneBinaryClassifiesENOEXEC(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("needs a POSIX execve") + } + dir := t.TempDir() + garbage := filepath.Join(dir, "garbage-cli") + // 0x7f 'E' 'L' 'F' with an impossible class byte: the kernel refuses it + // before any loader runs, and there is no shebang for sh to fall back to. + if err := os.WriteFile(garbage, append([]byte{0x7f, 'E', 'L', 'F', 0x09}, make([]byte, 64)...), 0o755); err != nil { + t.Fatal(err) + } + exits := filepath.Join(dir, "exits-cli") + if err := os.WriteFile(exits, []byte("#!/bin/sh\necho usage >&2\nexit 2\n"), 0o755); err != nil { + t.Fatal(err) + } + old := binaryPreflightTimeout + binaryPreflightTimeout = 10 * time.Second + t.Cleanup(func() { binaryPreflightTimeout = old }) + + f, bad := preflightOneBinary(context.Background(), "copilot", "copilot", garbage) + if !bad || !f.ExecFormat { + t.Fatalf("garbage ELF: want ENOEXEC finding, got bad=%v finding=%+v", bad, f) + } + if !strings.Contains(f.String(), "ENOEXEC") { + t.Fatalf("finding string lacks remedy: %q", f.String()) + } + if _, bad := preflightOneBinary(context.Background(), "codex", "codex", exits); bad { + t.Fatalf("a CLI that runs and exits non-zero must not be a finding") + } +} + +func TestPreflightBackendBinariesSkipsMissingCLIs(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + if got := PreflightBackendBinaries(context.Background(), []string{"copilot", "codex", "not-a-backend"}); len(got) != 0 { + t.Fatalf("nothing on PATH must yield no findings, got %+v", got) + } +} diff --git a/src/crux/agent/manager_credentials.go b/src/crux/agent/manager_credentials.go index 5fb1a2d898..b681e0a9c2 100644 --- a/src/crux/agent/manager_credentials.go +++ b/src/crux/agent/manager_credentials.go @@ -204,6 +204,26 @@ func (m *Manager) runCopilotDiagnostic(ctx context.Context, agent *AgentProcess) if output == "" { continue } + if paneShowsExecFormatError(strings.Split(output, "\n")) { + // The kernel refused the binary (#17729). No relaunch can fix + // that, and the restart-storm breaker would only turn it into + // 30-minute silences; park the agent with the remedy instead. + agent.LastError = execFormatLastError(binary) + m.logger.Error("diagnostic: copilot binary cannot be executed on this kernel (ENOEXEC); not relaunching", + "agent", agent.Name, "binary", binary, "output_tail", truncateStr(compactPaneTail(output, 8), 500), "remedy", execFormatRemedy) + if agent.UID > 0 { + killAgentProcesses(agent.UID, m.logger) + } + _ = m.tmuxCmd(agent, "kill-session", "-t", agent.tmuxSession).Run() + agent.State = StateFailed + m.audit(AuditAgentStartFailed, agent.Name, auditFields( + "outcome", "failure", + "backend", agent.effectiveBackend(), + "model", agent.effectiveModel(), + "error", agent.LastError, + )) + return + } if matchesAuthError(output) { agent.LastError = "auth token expired or invalid" // Prefer to RESTORE the stored token over merely clearing it: an diff --git a/src/crux/runtime/runtime.go b/src/crux/runtime/runtime.go index 5f3070e61e..d27a2c224a 100644 --- a/src/crux/runtime/runtime.go +++ b/src/crux/runtime/runtime.go @@ -26,6 +26,7 @@ import ( "fmt" "io" "log/slog" + "sort" "strings" "time" @@ -148,9 +149,32 @@ func newInProcess(o options) crux.Runtime { if o.sharedPool != nil { mopts = append(mopts, agent.WithSharedSessionPool(o.sharedPool)) } + // The in-process runtime is the one that execs agent CLIs on this host, + // so this is where a kernel that refuses one of them (ENOEXEC, #17729) + // gets said out loud once at boot — in the background, so a slow + // `--version` never delays the manager. + go agent.LogBackendBinaryPreflight(context.Background(), o.logger, backendsInUse(o.defs)) return agent.AsRuntime(agent.NewManagerWithOptions(o.defs, o.logger, o.project, mopts...)) } +// backendsInUse lists the distinct backends the agent definitions name, so +// the boot preflight probes the CLIs this hive will actually launch (and +// every installed CLI when no definitions were handed in). +func backendsInUse(defs map[string]config.AgentConfig) []string { + seen := map[string]bool{} + var out []string + for _, d := range defs { + b := strings.TrimSpace(d.Backend) + if b == "" || seen[b] { + continue + } + seen[b] = true + out = append(out, b) + } + sort.Strings(out) + return out +} + // HostOf returns rt's crux.Host when rt is the in-process implementation. // A sidecar runtime has no Host: its capabilities arrive over the API // (stage 2 and 3), so callers treat ok == false as "nothing to wire". From 2764dbca81915b2bd77dae55d1725d885d48f8e2 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 20:01:54 +0000 Subject: [PATCH 002/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.221.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.221.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/added-workflowmodel-marketplace.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-workflowmodel-marketplace.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 647f4ebfbb..f1a05a1eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.221.0) + +### Added + +- **workflow models:** RFC phase 7 — signed bundles (`pkg/workflowmodel/bundle`: model + compiled profile + sha256; import always re-verifies and recompiles locally), a hub marketplace catalogue (`/api/marketplace/*`: immutable `name@version` entries, ETag index, tiers built-in · curated · ingested-unreviewed · local-published, nightly/on-demand refresh of the embedded CNCF corpus publishing `ingested-YYYYMMDD.N`, curated versions only flagged `newerIngestAvailable`), spoke import/publish/proxy routes with `workflow.marketplace.{enabled,url,refresh_interval}`, and a Settings → Workflows → Marketplace view with Import, Publish and Refresh catalogue ([#17620](https://github.com/hivecommons/hive/issues/17620)) + ## 2026-11-11 (v5.220.0) ### Added diff --git a/changelog.d/added-workflowmodel-marketplace.md b/changelog.d/added-workflowmodel-marketplace.md deleted file mode 100644 index bae3f3071c..0000000000 --- a/changelog.d/added-workflowmodel-marketplace.md +++ /dev/null @@ -1 +0,0 @@ -- **workflow models:** RFC phase 7 — signed bundles (`pkg/workflowmodel/bundle`: model + compiled profile + sha256; import always re-verifies and recompiles locally), a hub marketplace catalogue (`/api/marketplace/*`: immutable `name@version` entries, ETag index, tiers built-in · curated · ingested-unreviewed · local-published, nightly/on-demand refresh of the embedded CNCF corpus publishing `ingested-YYYYMMDD.N`, curated versions only flagged `newerIngestAvailable`), spoke import/publish/proxy routes with `workflow.marketplace.{enabled,url,refresh_interval}`, and a Settings → Workflows → Marketplace view with Import, Publish and Refresh catalogue ([#17620](https://github.com/hivecommons/hive/issues/17620)) From a285b5041c3f0b15f819b6e9c35078aeab3b34a3 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:16:43 +0000 Subject: [PATCH 003/223] fix: warn once per agent about user identity in agent gitconfig (#17742) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-gitconfig-identity-warning-once.md | 1 + src/crux/agent/git_identity_warn_dedupe.go | 37 +++++++++++++++++++ src/crux/agent/interactive_home.go | 3 ++ .../agent/interactive_home_gitconfig_test.go | 34 +++++++++++++++++ src/crux/agent/manager.go | 3 +- 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-gitconfig-identity-warning-once.md create mode 100644 src/crux/agent/git_identity_warn_dedupe.go diff --git a/changelog.d/fixed-gitconfig-identity-warning-once.md b/changelog.d/fixed-gitconfig-identity-warning-once.md new file mode 100644 index 0000000000..55515cc892 --- /dev/null +++ b/changelog.d/fixed-gitconfig-identity-warning-once.md @@ -0,0 +1 @@ +- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). diff --git a/src/crux/agent/git_identity_warn_dedupe.go b/src/crux/agent/git_identity_warn_dedupe.go new file mode 100644 index 0000000000..b3d04f9913 --- /dev/null +++ b/src/crux/agent/git_identity_warn_dedupe.go @@ -0,0 +1,37 @@ +package agent + +import ( + "crypto/sha256" + "os" + "sync" +) + +// strayGitIdentityWarned remembers, per (agent, config path), the content hash +// of the git config the stray-identity WARN was last emitted for. The launch +// check runs every cycle; without this the identical informational WARN would +// repeat and trip the repeated-error selfreport (hivecommons/hive#17703). +// State is in-memory, so a hive restart warns once more. +type strayGitIdentityWarned struct { + mu sync.Mutex + seen map[string][sha256.Size]byte +} + +// shouldWarn reports whether the finding at path is new for agentName: true on +// first sight and whenever the file content changed since the last warning. +func (s *strayGitIdentityWarned) shouldWarn(agentName, path string) bool { + var sum [sha256.Size]byte + if data, err := os.ReadFile(path); err == nil { + sum = sha256.Sum256(data) + } + key := agentName + "\x00" + path + s.mu.Lock() + defer s.mu.Unlock() + if prev, ok := s.seen[key]; ok && prev == sum { + return false + } + if s.seen == nil { + s.seen = make(map[string][sha256.Size]byte) + } + s.seen[key] = sum + return true +} diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index 02e5974e1e..114e404d1e 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -521,6 +521,9 @@ func (m *Manager) warnOnStrayGitIdentity(agentName, home string) { if !gitConfigDeclaresUserIdentity(path) { continue } + if !m.strayGitIdentity.shouldWarn(agentName, path) { + continue + } m.logger.Warn("agent git config declares a user identity outside the system /etc/gitconfig; commits still use the pinned per-lane GIT_AUTHOR_*/GIT_COMMITTER_* identity, but `git config user.email` reads this shared value instead (hivecommons/hive#9478)", "agent", agentName, "config", path) } diff --git a/src/crux/agent/interactive_home_gitconfig_test.go b/src/crux/agent/interactive_home_gitconfig_test.go index 6140b0c4cd..c2014e8cab 100644 --- a/src/crux/agent/interactive_home_gitconfig_test.go +++ b/src/crux/agent/interactive_home_gitconfig_test.go @@ -132,6 +132,40 @@ func TestWarnOnStrayGitIdentity(t *testing.T) { } } +// TestWarnOnStrayGitIdentity_OncePerAgentAndContent: the check runs every +// cycle, so an unchanged file must warn once, a changed file or another agent +// must warn again (hivecommons/hive#17703). +func TestWarnOnStrayGitIdentity_OncePerAgentAndContent(t *testing.T) { + const marker = "declares a user identity" + home := t.TempDir() + path := filepath.Join(home, ".gitconfig") + write := func(body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("[user]\n\temail = a@example.com\n") + + m, logBuf := gitConfigHomeManager(t) + m.warnOnStrayGitIdentity("scanner", home) + m.warnOnStrayGitIdentity("scanner", home) + if got := strings.Count(logBuf.String(), marker); got != 1 { + t.Fatalf("repeat with unchanged config warned %d times, want 1: %s", got, logBuf.String()) + } + + write("[user]\n\temail = b@example.com\n") + m.warnOnStrayGitIdentity("scanner", home) + if got := strings.Count(logBuf.String(), marker); got != 2 { + t.Fatalf("changed config warned %d times total, want 2", got) + } + + m.warnOnStrayGitIdentity("reviewer", home) + if got := strings.Count(logBuf.String(), marker); got != 3 { + t.Fatalf("different agent warned %d times total, want 3", got) + } +} + func TestGitConfigDeclaresUserIdentity(t *testing.T) { dir := t.TempDir() cases := []struct { diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 9a03a1dcc0..64b76f38c6 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -552,7 +552,8 @@ type Manager struct { // own mutex: the pane pollers that consult it must not take m.mu. rateLimitRelaunchMu sync.Mutex lastRateLimitRelaunch time.Time - cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go + cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go + strayGitIdentity strayGitIdentityWarned // warn-once state, see git_identity_warn_dedupe.go // thrashMu guards thrash — its own mutex, NEVER m.mu: the breaker runs on // the output-capture goroutines, and taking m.mu there risks startup From c95518e54cc2a6d063e76550e7dff3c73257233d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:26:56 +0000 Subject: [PATCH 004/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20name=20ENOEXEC=20?= =?UTF-8?q?instead=20of=20dumping=20binary=20stderr=20in=20model=20discove?= =?UTF-8?q?ry=20error=20(#17756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: name ENOEXEC instead of dumping binary stderr in model discovery error Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 fix: import unicode/utf8 and keep ANSI-coloured stderr out of the ENOEXEC check cli_models.go used utf8.ValidString without importing unicode/utf8, so the dashboard package failed to build. isBinaryStderr also treated ESC (0x1b) as a binary control byte, so coloured stderr would have been mislabelled as an ENOEXEC dump; ESC is now allowed alongside the whitespace controls, with a regression test. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/fixed-discovery-binary-stderr.md | 1 + src/pkg/dashboard/cli_models.go | 19 +++++++++++++++++ .../dashboard/cli_models_last_good_test.go | 21 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 changelog.d/fixed-discovery-binary-stderr.md diff --git a/changelog.d/fixed-discovery-binary-stderr.md b/changelog.d/fixed-discovery-binary-stderr.md new file mode 100644 index 0000000000..18be9f2d91 --- /dev/null +++ b/changelog.d/fixed-discovery-binary-stderr.md @@ -0,0 +1 @@ +- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). diff --git a/src/pkg/dashboard/cli_models.go b/src/pkg/dashboard/cli_models.go index c8342e6eae..81000fd55a 100644 --- a/src/pkg/dashboard/cli_models.go +++ b/src/pkg/dashboard/cli_models.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/claude" @@ -1613,12 +1614,30 @@ func copilotSDKHelperExecErr(ctx context.Context, err error, stderr string) erro errCopilotSDKHelperTimeout, copilotSDKProbeTimeout, filepath.Base(copilotSDKHelperPath)) } msg := strings.TrimSpace(stderr) + if isBinaryStderr(msg) { + return fmt.Errorf("sdk helper: %w (stderr was binary: the kernel refused to exec the Copilot CLI ELF and sh tried to run it as a script — ENOEXEC, see the copilot exec shim; not a model-catalog fault)", err) + } if len(msg) > copilotSDKStderrLogLimit { msg = msg[:copilotSDKStderrLogLimit] } return fmt.Errorf("sdk helper: %w (stderr: %s)", err, msg) } +// isBinaryStderr reports stderr that is not text: invalid UTF-8 or control +// bytes other than whitespace and ESC (ANSI colour), as when sh dumps an ELF +// it could not exec. +func isBinaryStderr(s string) bool { + if !utf8.ValidString(s) { + return true + } + for _, r := range s { + if r < 0x20 && r != '\n' && r != '\r' && r != '\t' && r != 0x1b { + return true + } + } + return false +} + // copilotSDKModel mirrors one entry of the helper's stdout JSON. The helper // also emits name/efforts/defaultEffort for future use; discovery only needs // the id and the policy gate today. diff --git a/src/pkg/dashboard/cli_models_last_good_test.go b/src/pkg/dashboard/cli_models_last_good_test.go index 97c8434bba..79edcf6b26 100644 --- a/src/pkg/dashboard/cli_models_last_good_test.go +++ b/src/pkg/dashboard/cli_models_last_good_test.go @@ -11,6 +11,7 @@ import ( "sync" "testing" "time" + "unicode/utf8" ) // newLastGoodTestCache builds a cache with a pinned clock and an isolated @@ -63,6 +64,26 @@ func TestCopilotSDKHelperExecErr_OtherKeepsStderr(t *testing.T) { } } +// TestCopilotSDKHelperExecErr_BinaryStderrNamed: an ELF dumped as stderr (#17735) +// must not reach the self-report as garbage; it is named as an ENOEXEC. +func TestCopilotSDKHelperExecErr_BinaryStderrNamed(t *testing.T) { + err := copilotSDKHelperExecErr(context.Background(), errors.New("exit status 1"), "copilot: 1: \x7fELF\x02\x01\x00\xff\xfe\x06") + got := err.Error() + if !strings.Contains(got, "ENOEXEC") || strings.Contains(got, "ELF\x02") || !utf8.ValidString(got) { + t.Fatalf("binary stderr not summarised: %q", got) + } +} + +// TestCopilotSDKHelperExecErr_ANSIStderrKept: coloured stderr is text, not an +// ELF dump — it must be passed through rather than mislabelled as ENOEXEC. +func TestCopilotSDKHelperExecErr_ANSIStderrKept(t *testing.T) { + err := copilotSDKHelperExecErr(context.Background(), errors.New("exit status 1"), "\x1b[31merror:\x1b[0m auth required\n") + got := err.Error() + if strings.Contains(got, "ENOEXEC") || !strings.Contains(got, "auth required") { + t.Fatalf("ANSI stderr mislabelled: %q", got) + } +} + // TestQueryCLIModels_SDKFailureServesLastGoodCatalog is the #11865 flap: a // clean SDK probe, then one that dies. The served list must stay the CLI's // catalog — authoritative, labelled stale with the helper failure as the From 583934ab2df132646d92c96e5384176e968746b6 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 14:45:31 -0700 Subject: [PATCH 005/223] =?UTF-8?q?=E2=9C=A8=20crux:=20Substrate=20as=20th?= =?UTF-8?q?e=204th=20sandbox=20runtime=20(opt-in,=20preflight-gated)=20(#1?= =?UTF-8?q?7741)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: Substrate as the 4th sandbox runtime (opt-in, preflight-gated) Add Agent Substrate (CNCF Sandbox) as sandbox.runtime: substrate behind the existing sandbox.Launcher seam, mirroring the OpenShell runtime (ADR-0026 phase 2). - crux/sandbox/substrate*.go: SubstrateLauncher (ActorTemplate + Actor per kick, atenet-egress policy derived from the proxy host set + allow_hosts, workspace push/run/pull through an in-actor bridge over the atenet-router mTLS CONNECT tunnel, snapshot_on_idle suspends instead of deleting), real gRPC client constructed lazily, Control API preflight registered into crux/config. - crux/sandbox/substrateapi: trimmed, wire-compatible proto subset of ate.control.v1 generated in-repo (no go.ate.dev/substrate import: it requires Go 1.27 and is very large). Zero new Go modules. - crux/sandbox/substratebridge: the HTTP bridge served inside the actor. - crux/config/agent_config_substrate.go: sandbox.substrate.* leaves with per-agent merge; pkg/config gate warnings (digest-pinned image). - Settings parity allowlist entries for the new leaves (#17699). Fixes #17697 Part of #17695 Signed-off-by: Andrew Anderson * changelog: fragment for #17741 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🌱 changelog: make the substrate fragment an entry bullet Signed-off-by: Andrew Anderson * 🌱 changelog: drop the auto-generated duplicate fragment Signed-off-by: Andrew Anderson * 🌱 substrateapi: drop a never-true nil comparison flagged by staticcheck Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../added-substrate-sandbox-runtime.md | 1 + src/crux/agent/manager.go | 3 + src/crux/agent/manager_sandbox.go | 14 +- src/crux/agent/manager_sandbox_substrate.go | 78 ++ .../agent/sandbox_substrate_runtime_test.go | 141 ++++ src/crux/config/agent_config.go | 8 +- src/crux/config/agent_config_substrate.go | 179 +++++ .../config/agent_config_substrate_test.go | 121 ++++ src/crux/go.mod | 4 +- src/crux/sandbox/substrate.go | 387 ++++++++++ src/crux/sandbox/substrate_bridge.go | 107 +++ src/crux/sandbox/substrate_client.go | 292 ++++++++ src/crux/sandbox/substrate_client_test.go | 681 ++++++++++++++++++ src/crux/sandbox/substrate_preflight.go | 125 ++++ src/crux/sandbox/substrate_spec.go | 234 ++++++ src/crux/sandbox/substrate_test.go | 652 +++++++++++++++++ src/crux/sandbox/substrateapi/actor.pb.go | 586 +++++++++++++++ src/crux/sandbox/substrateapi/actor.proto | 79 ++ src/crux/sandbox/substrateapi/actor_rpc.pb.go | 572 +++++++++++++++ src/crux/sandbox/substrateapi/actor_rpc.proto | 72 ++ src/crux/sandbox/substrateapi/buf.gen.yaml | 8 + src/crux/sandbox/substrateapi/buf.yaml | 3 + src/crux/sandbox/substrateapi/common.pb.go | 428 +++++++++++ src/crux/sandbox/substrateapi/common.proto | 59 ++ src/crux/sandbox/substrateapi/container.pb.go | 401 +++++++++++ src/crux/sandbox/substrateapi/container.proto | 54 ++ src/crux/sandbox/substrateapi/control.pb.go | 135 ++++ src/crux/sandbox/substrateapi/control.proto | 48 ++ .../sandbox/substrateapi/control_grpc.pb.go | 490 +++++++++++++ src/crux/sandbox/substrateapi/egress.pb.go | 553 ++++++++++++++ src/crux/sandbox/substrateapi/egress.proto | 68 ++ src/crux/sandbox/substrateapi/generate.sh | 6 + .../sandbox/substrateapi/substrateapi_test.go | 364 ++++++++++ src/crux/sandbox/substrateapi/template.pb.go | 616 ++++++++++++++++ src/crux/sandbox/substrateapi/template.proto | 76 ++ src/crux/sandbox/substrateapi/worker.pb.go | 499 +++++++++++++ src/crux/sandbox/substrateapi/worker.proto | 67 ++ src/crux/sandbox/substratebridge/bridge.go | 338 +++++++++ .../sandbox/substratebridge/bridge_test.go | 394 ++++++++++ .../config/agent_sandbox_substrate_test.go | 115 +++ src/pkg/config/config.go | 19 +- src/pkg/config/crux_aliases.go | 2 + .../dashboard/settings_parity_allowlist.txt | 38 + 43 files changed, 9112 insertions(+), 5 deletions(-) create mode 100644 changelog.d/added-substrate-sandbox-runtime.md create mode 100644 src/crux/agent/manager_sandbox_substrate.go create mode 100644 src/crux/agent/sandbox_substrate_runtime_test.go create mode 100644 src/crux/config/agent_config_substrate.go create mode 100644 src/crux/config/agent_config_substrate_test.go create mode 100644 src/crux/sandbox/substrate.go create mode 100644 src/crux/sandbox/substrate_bridge.go create mode 100644 src/crux/sandbox/substrate_client.go create mode 100644 src/crux/sandbox/substrate_client_test.go create mode 100644 src/crux/sandbox/substrate_preflight.go create mode 100644 src/crux/sandbox/substrate_spec.go create mode 100644 src/crux/sandbox/substrate_test.go create mode 100644 src/crux/sandbox/substrateapi/actor.pb.go create mode 100644 src/crux/sandbox/substrateapi/actor.proto create mode 100644 src/crux/sandbox/substrateapi/actor_rpc.pb.go create mode 100644 src/crux/sandbox/substrateapi/actor_rpc.proto create mode 100644 src/crux/sandbox/substrateapi/buf.gen.yaml create mode 100644 src/crux/sandbox/substrateapi/buf.yaml create mode 100644 src/crux/sandbox/substrateapi/common.pb.go create mode 100644 src/crux/sandbox/substrateapi/common.proto create mode 100644 src/crux/sandbox/substrateapi/container.pb.go create mode 100644 src/crux/sandbox/substrateapi/container.proto create mode 100644 src/crux/sandbox/substrateapi/control.pb.go create mode 100644 src/crux/sandbox/substrateapi/control.proto create mode 100644 src/crux/sandbox/substrateapi/control_grpc.pb.go create mode 100644 src/crux/sandbox/substrateapi/egress.pb.go create mode 100644 src/crux/sandbox/substrateapi/egress.proto create mode 100755 src/crux/sandbox/substrateapi/generate.sh create mode 100644 src/crux/sandbox/substrateapi/substrateapi_test.go create mode 100644 src/crux/sandbox/substrateapi/template.pb.go create mode 100644 src/crux/sandbox/substrateapi/template.proto create mode 100644 src/crux/sandbox/substrateapi/worker.pb.go create mode 100644 src/crux/sandbox/substrateapi/worker.proto create mode 100644 src/crux/sandbox/substratebridge/bridge.go create mode 100644 src/crux/sandbox/substratebridge/bridge_test.go create mode 100644 src/pkg/config/agent_sandbox_substrate_test.go diff --git a/changelog.d/added-substrate-sandbox-runtime.md b/changelog.d/added-substrate-sandbox-runtime.md new file mode 100644 index 0000000000..f9c190943b --- /dev/null +++ b/changelog.d/added-substrate-sandbox-runtime.md @@ -0,0 +1 @@ +- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 64b76f38c6..92102c3b9d 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -767,6 +767,9 @@ type Manager struct { // sandboxOpenShellLauncherFactory is the same seam for // sandbox.runtime: openshell (ADR-0026). sandboxOpenShellLauncherFactory func(config.SandboxOpenShellConfig) sandbox.Launcher + // sandboxSubstrateLauncherFactory is the same seam for + // sandbox.runtime: substrate (ADR-0026 phase 2). + sandboxSubstrateLauncherFactory func(config.SandboxSubstrateConfig) sandbox.Launcher sandboxRunner sandboxCommandRunner // sandboxRunPersister overrides persistSandboxRun in tests. sandboxRunPersister func(workDir string, entry SandboxRunHistoryEntry) error diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index d0d4d31235..93906a0079 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -18,6 +18,7 @@ import ( func init() { config.SetOpenShellPreflight(openShellPreflightWarnings) + config.SetSubstratePreflight(substratePreflightWarnings) } // openShellPreflightWarnings checks each distinct gateway the openshell @@ -112,7 +113,7 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) // does, instead of failing it after the state machine has moved. runtime := agent.Config.SandboxRuntime(m.sandboxConfig) if !config.ValidSandboxRuntimes[runtime] { - return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell) + return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell, config.SandboxRuntimeSubstrate) } switch runtime { case config.SandboxRuntimeJob: @@ -129,6 +130,15 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) if effectiveBackend(agent) == "cline" { return "", fmt.Errorf("agent %s: the Cline backend is not supported on the openshell sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) } + case config.SandboxRuntimeSubstrate: + // ADR-0026 phase 2: same contract as openshell — inert unless + // selected, refused before the state machine moves. + if err := SubstrateOptionsFromConfig(agent.Config.SandboxSubstrate(m.sandboxConfig)).Validate(); err != nil { + return "", fmt.Errorf("agent %s sandbox substrate runtime: %w", agent.Name, err) + } + if effectiveBackend(agent) == "cline" { + return "", fmt.Errorf("agent %s: the Cline backend is not supported on the substrate sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) + } } if effectiveBackend(agent) == "cline" { if err := validateClineSandbox(agent, runtime, agent.Config.SandboxJob(m.sandboxConfig)); err != nil { @@ -149,6 +159,8 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er launcher = m.jobLauncherLocked(agent.Config.SandboxJob(m.sandboxConfig)) case config.SandboxRuntimeOpenShell: launcher = m.openshellLauncherLocked(agent.Config.SandboxOpenShell(m.sandboxConfig)) + case config.SandboxRuntimeSubstrate: + launcher = m.substrateLauncherLocked(agent.Config.SandboxSubstrate(m.sandboxConfig)) } repo := m.project.PrimaryRepo() now := time.Now() diff --git a/src/crux/agent/manager_sandbox_substrate.go b/src/crux/agent/manager_sandbox_substrate.go new file mode 100644 index 0000000000..4d535bf238 --- /dev/null +++ b/src/crux/agent/manager_sandbox_substrate.go @@ -0,0 +1,78 @@ +package agent + +import ( + "context" + "os" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// sandbox.runtime: substrate wiring (ADR-0026 phase 2); the sibling of the +// openshell helpers in manager_sandbox.go. + +// substratePreflightWarnings checks each distinct control plane the +// substrate agents resolve to, once. +func substratePreflightWarnings(blocks []config.SandboxSubstrateConfig) []string { + var out []string + seen := map[string]bool{} + for _, b := range blocks { + opts := SubstrateOptionsFromConfig(b) + key := opts.Endpoint + "\x00" + opts.Atespace + "\x00" + opts.Token + if seen[key] { + continue + } + seen[key] = true + out = append(out, sandbox.DefaultSubstratePreflighter.Check(context.Background(), opts)...) + } + return out +} + +func (m *Manager) setSandboxSubstrateLauncherFactoryForTest(f func(config.SandboxSubstrateConfig) sandbox.Launcher) { + m.mu.Lock() + defer m.mu.Unlock() + m.sandboxSubstrateLauncherFactory = f +} + +// substrateLauncherLocked returns the launcher for a sandbox.runtime: +// substrate kick: the injected factory's, or a real Control-API-backed +// sandbox.SubstrateLauncher. +func (m *Manager) substrateLauncherLocked(cfg config.SandboxSubstrateConfig) sandbox.Launcher { + if m.sandboxSubstrateLauncherFactory != nil { + return m.sandboxSubstrateLauncherFactory(cfg) + } + return &sandbox.SubstrateLauncher{Options: SubstrateOptionsFromConfig(cfg)} +} + +// SubstrateOptionsFromConfig converts the operator's substrate block into +// the launcher's options. A token named by token_env is read from the +// environment so it never sits in hive.yaml; otherwise the launcher reads +// the projected token file per request. +func SubstrateOptionsFromConfig(cfg config.SandboxSubstrateConfig) sandbox.SubstrateOptions { + opts := sandbox.SubstrateOptions{ + Endpoint: cfg.Endpoint, + Atespace: cfg.Atespace, + TokenFile: cfg.TokenFile, + CAFile: cfg.CAFile, + InsecureTLS: cfg.InsecureTLS, + Timeout: time.Duration(cfg.TimeoutS) * time.Second, + RouterURL: cfg.RouterURL, + RouterClientBundle: cfg.RouterClientBundle, + RouterCAFile: cfg.RouterCAFile, + BridgePort: cfg.BridgePort, + Image: cfg.Image, + Command: append([]string(nil), cfg.BridgeCommand...), + CPUMillicores: cfg.CPUMillicores, + MemoryBytes: cfg.MemoryBytes, + WorkerSelector: cfg.WorkerSelector, + SnapshotOnIdle: cfg.SnapshotOnIdle, + AllowHosts: append([]string(nil), cfg.AllowHosts...), + PolicyProfile: cfg.PolicyProfile, + } + if env := strings.TrimSpace(cfg.TokenEnv); env != "" { + opts.Token = os.Getenv(env) + } + return opts +} diff --git a/src/crux/agent/sandbox_substrate_runtime_test.go b/src/crux/agent/sandbox_substrate_runtime_test.go new file mode 100644 index 0000000000..dfd75d04fa --- /dev/null +++ b/src/crux/agent/sandbox_substrate_runtime_test.go @@ -0,0 +1,141 @@ +package agent + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// The Substrate runtime (ADR-0026 phase 2) routes a sandboxed kick through +// the launcher built from the agent's substrate block, refuses a kick whose +// options are invalid before the state machine moves, and stays inert for +// every agent that did not select it. + +const substrateTestImage = "ghcr.io/hivecommons/sandbox@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func substrateRuntimeManager(t *testing.T, override *config.AgentSandboxOverride, global config.AgentSandboxConfig) (*Manager, *int32, *int32) { + t.Helper() + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude", Sandbox: override}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + if global.WorkspaceDir == "" { + global.WorkspaceDir = t.TempDir() + } + m.SetSandboxConfig(global) + var podmanRuns, substrateRuns int32 + m.SetSandboxLauncher(countingLauncher{runs: &podmanRuns}) + m.setSandboxSubstrateLauncherFactoryForTest(func(sb config.SandboxSubstrateConfig) sandbox.Launcher { + if sb.Atespace != "hive-test" { + t.Errorf("factory got atespace=%q", sb.Atespace) + } + return countingLauncher{runs: &substrateRuns} + }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + return m, &podmanRuns, &substrateRuns +} + +func TestSubstrateRuntimeKickUsesTheSubstrateLauncher(t *testing.T) { + yes := true + m, podman, sbRuns := substrateRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Substrate: &config.SandboxSubstrateConfig{Atespace: "hive-test", Image: substrateTestImage}}) + if err := m.SendKick("deps", "bump the pinned dependency"); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "deps") + if got := atomic.LoadInt32(sbRuns); got != 1 { + t.Errorf("substrate launcher runs = %d, want 1", got) + } + if got := atomic.LoadInt32(podman); got != 0 { + t.Errorf("podman launcher ran %d time(s) for a substrate-runtime agent", got) + } +} + +func TestSubstrateRuntimeRefusesKickWithTagOnlyImage(t *testing.T) { + yes := true + m, _, sbRuns := substrateRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Substrate: &config.SandboxSubstrateConfig{Atespace: "hive-test", Image: "ghcr.io/x/y:latest"}}) + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("SendKick error = %v, want a refusal naming the digest rule", err) + } + if got := atomic.LoadInt32(sbRuns); got != 0 { + t.Errorf("launcher ran %d time(s) despite the refusal", got) + } + if st := m.AllStatuses()["deps"].State; st == StateRunning { + t.Errorf("agent marked running after a refused kick") + } +} + +func TestSubstrateRuntimeRefusesCline(t *testing.T) { + yes := true + cfg := map[string]config.AgentConfig{"deps": {Backend: "cline", Sandbox: &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true, Image: "dev-image", WorkspaceDir: t.TempDir(), + Substrate: &config.SandboxSubstrateConfig{Image: substrateTestImage}}) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "Cline") { + t.Fatalf("SendKick error = %v, want the Cline refusal", err) + } +} + +func TestSubstrateOptionsFromConfig(t *testing.T) { + t.Setenv("SUBSTRATE_TEST_TOKEN", "sekrit") + opts := SubstrateOptionsFromConfig(config.SandboxSubstrateConfig{ + Endpoint: "api.ate-system.svc:443", + Atespace: "hive", + PolicyProfile: "strict", + TimeoutS: 90, + TokenEnv: "SUBSTRATE_TEST_TOKEN", + Image: substrateTestImage, + BridgeCommand: []string{"/bin/bridge"}, + AllowHosts: []string{"litellm.example.com"}, + WorkerSelector: map[string]string{"class": "gvisor"}, + SnapshotOnIdle: true, + }) + if opts.Token != "sekrit" || opts.Atespace != "hive" || opts.PolicyProfile != "strict" || opts.Timeout != 90*time.Second || !opts.SnapshotOnIdle { + t.Errorf("opts = %+v", opts) + } + if len(opts.Command) != 1 || opts.Command[0] != "/bin/bridge" || opts.WorkerSelector["class"] != "gvisor" { + t.Errorf("command/selector = %+v", opts) + } + if err := opts.Validate(); err != nil { + t.Errorf("Validate: %v", err) + } +} + +func TestSubstrateLauncherDefaultsToControlAPI(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{}, quietTestLogger(), ProjectContext{}) + l := m.substrateLauncherLocked(config.SandboxSubstrateConfig{Atespace: "hive"}) + if _, ok := l.(*sandbox.SubstrateLauncher); !ok { + t.Fatalf("launcher = %T, want *sandbox.SubstrateLauncher", l) + } +} + +func TestSubstratePreflightWarningsDedupesControlPlanes(t *testing.T) { + // Two agents on the same control plane probe once. A CA file that does + // not exist fails the client constructor before any network I/O, which + // is reported as the control plane being unreachable. + blocks := []config.SandboxSubstrateConfig{ + {Endpoint: "127.0.0.1:1", Atespace: "dedupe-" + t.Name(), CAFile: "/nonexistent/ca.crt"}, + {Endpoint: "127.0.0.1:1", Atespace: "dedupe-" + t.Name(), CAFile: "/nonexistent/ca.crt"}, + } + got := substratePreflightWarnings(blocks) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Fatalf("warnings = %q, want exactly one unreachable warning for the shared control plane", got) + } +} diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index de810c3a0b..c3b9d573e8 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -10,10 +10,11 @@ const ( SandboxRuntimePodman = "podman" SandboxRuntimeJob = "job" SandboxRuntimeOpenShell = "openshell" + SandboxRuntimeSubstrate = "substrate" ) // ValidSandboxRuntimes is the closed set a config may name. -var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true} +var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true, SandboxRuntimeSubstrate: true} // SandboxJobConfig is everything the Job runtime needs beyond the image // (#6311). The pod-template fields pass through to Kubernetes verbatim. @@ -499,6 +500,8 @@ type AgentSandboxConfig struct { Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` // OpenShell holds the settings for sandbox.runtime: openshell (ADR-0026). OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` + // Substrate holds the settings for sandbox.runtime: substrate (ADR-0026 phase 2). + Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish gates what a sandboxed run may publish; see SandboxPublishConfig. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` } @@ -540,6 +543,9 @@ type AgentSandboxOverride struct { // OpenShell, when set, overrides the global openshell block field by // field for this agent; see AgentConfig.SandboxOpenShell. OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` + // Substrate, when set, overrides the global substrate block field by + // field for this agent; see AgentConfig.SandboxSubstrate. + Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish, when set, replaces the global publish preflight for this // agent; an empty verify list here turns the preflight off for it. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` diff --git a/src/crux/config/agent_config_substrate.go b/src/crux/config/agent_config_substrate.go new file mode 100644 index 0000000000..327eb6959b --- /dev/null +++ b/src/crux/config/agent_config_substrate.go @@ -0,0 +1,179 @@ +package config + +import ( + "strings" + "sync" +) + +var ( + substratePreflightMu sync.RWMutex + substratePreflight func([]SandboxSubstrateConfig) []string +) + +// SetSubstratePreflight registers the cluster-prerequisite check +// AgentSandboxGateWarnings runs for substrate-runtime agents. This package +// (and pkg/config above it) cannot import the Substrate client or +// crux/sandbox, so the code that owns the Control API client plugs itself in +// here. Passing nil unregisters it. +func SetSubstratePreflight(f func([]SandboxSubstrateConfig) []string) { + substratePreflightMu.Lock() + defer substratePreflightMu.Unlock() + substratePreflight = f +} + +// SubstratePreflightWarnings runs the registered preflight against the given +// substrate blocks; with none registered it reports nothing. +func SubstratePreflightWarnings(cfgs []SandboxSubstrateConfig) []string { + substratePreflightMu.RLock() + f := substratePreflight + substratePreflightMu.RUnlock() + if f == nil || len(cfgs) == 0 { + return nil + } + return f(cfgs) +} + +// SandboxSubstrateConfig is everything the Agent Substrate runtime needs +// beyond the agent's kick (ADR-0026 phase 2). Hive talks to Substrate's +// Control API (ate-api-server) over gRPC; Substrate owns the gVisor/microVM +// sandbox, the snapshot store and the atenet-egress gateway, so most of what +// the Job runtime spells out as a pod template is an ActorTemplate hive +// writes per kick from these settings. +// +// The workspace is NOT shared the way the Job and OpenShell runtimes share +// it: Substrate cannot mount an existing PersistentVolumeClaim, only a +// per-actor durableDir or a CSI volume it provisions itself. Hive therefore +// copies the per-kick workspace into the actor and back out through the +// bridge the sandbox image runs (crux substrate-bridge), reached over the +// atenet-router mTLS CONNECT tunnel. The local workspace stays the source of +// truth after the kick, so the push broker and PR flow are unchanged. +// +// AllowHosts is the egress allowlist handed to Substrate as the actor's +// EgressPolicy: the hosts hive's own MITM proxy (ADR-0002) would let a Podman +// sandbox reach — github.com, api.github.com — plus whatever the operator +// adds. Nothing else is reachable from the actor; Substrate default-denies. +// Credentials follow the Job rule: none of hive's enter it. Substrate's own +// egress credential injection is complementary and not used by hive. +type SandboxSubstrateConfig struct { + // Endpoint is the Control API address (host:port, or an http(s):// URL + // whose scheme picks TLS). Default api.ate-system.svc:443. + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // Atespace is the Substrate atespace (its tenant namespace, not a + // Kubernetes namespace) actors are created in. Default "hive". + Atespace string `yaml:"atespace,omitempty" json:"atespace,omitempty"` + // TokenEnv names the environment variable holding a Control API bearer + // token. Empty reads TokenFile instead. + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` + // TokenFile is the projected ServiceAccount token (audience + // api.ate-system.svc) hive authenticates with. Default + // /var/run/secrets/ate/token. + TokenFile string `yaml:"token_file,omitempty" json:"token_file,omitempty"` + // CAFile is the PEM bundle the Control API serving certificate chains + // to (the servicedns ClusterTrustBundle projection). Empty uses + // /var/run/secrets/ate/ca.crt when present, else the system roots. + CAFile string `yaml:"ca_file,omitempty" json:"ca_file,omitempty"` + // InsecureTLS skips Control API and router certificate verification. + // Dev only. + InsecureTLS bool `yaml:"insecure_tls,omitempty" json:"insecure_tls,omitempty"` + // TimeoutS bounds one actor from create to delete. Non-positive uses + // the sandbox timeout (agent_sandbox.timeout_s) hive already applies to + // the kick. + TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` + // RouterURL is the atenet-router CONNECT listener hive reaches the + // actor's bridge through. Default https://atenet-router.ate-system.svc:8444. + RouterURL string `yaml:"router_url,omitempty" json:"router_url,omitempty"` + // RouterClientBundle and RouterCAFile are the podCertificate bundle the + // router authenticates hive with and the CA its serving certificate + // chains to. Defaults /run/router-client/credential-bundle.pem and + // /run/router-client/router-ca.pem. + RouterClientBundle string `yaml:"router_client_bundle,omitempty" json:"router_client_bundle,omitempty"` + RouterCAFile string `yaml:"router_ca_file,omitempty" json:"router_ca_file,omitempty"` + // BridgePort is the port the in-actor bridge listens on. Default 8477. + BridgePort int `yaml:"bridge_port,omitempty" json:"bridge_port,omitempty"` + // Image is the digest-pinned sandbox image (…@sha256:…) actors run. + // Substrate refuses tag-only references, so this overrides the agent + // sandbox image, which is usually a tag. + Image string `yaml:"image,omitempty" json:"image,omitempty"` + // BridgeCommand is the actor entrypoint that serves the bridge. Empty + // uses the crux binary's substrate-bridge subcommand. + BridgeCommand []string `yaml:"bridge_command,omitempty" json:"bridge_command,omitempty"` + // CPUMillicores and MemoryBytes are the ActorTemplate resource limits; + // Substrate requires both. Defaults 2000 and 4 GiB. + CPUMillicores int64 `yaml:"cpu_millicores,omitempty" json:"cpu_millicores,omitempty"` + MemoryBytes int64 `yaml:"memory_bytes,omitempty" json:"memory_bytes,omitempty"` + // WorkerSelector pins actors to WorkerPools carrying these labels; the + // pool decides the sandbox class (gvisor or microvm). Empty means any + // pool. + WorkerSelector map[string]string `yaml:"worker_selector,omitempty" json:"worker_selector,omitempty"` + // SnapshotOnIdle suspends the actor (asynchronous snapshot) after the + // kick instead of deleting it, so Substrate keeps a resumable image. + // Default false: a kick is one-shot and snapshots cost object storage. + SnapshotOnIdle bool `yaml:"snapshot_on_idle,omitempty" json:"snapshot_on_idle,omitempty"` + // AllowHosts are additional egress hosts (host or host:port, leading + // *. wildcard allowed) the actor may reach, on top of the GitHub hosts + // hive always allows. + AllowHosts []string `yaml:"allow_hosts,omitempty" json:"allow_hosts,omitempty"` + // PolicyProfile names an operator-side profile; hive records it on the + // template so pools and in-actor tooling can act on it. + PolicyProfile string `yaml:"policy_profile,omitempty" json:"policy_profile,omitempty"` +} + +// SandboxSubstrate returns the effective Substrate runtime settings. The +// endpoint, atespace, credentials, TLS, router and bridge fields are cluster +// facts, so a per-agent block falls back to the global one for each it +// leaves empty; the image, resources, pool selector, snapshot choice and +// extra egress hosts are per-agent choices and replace the global ones +// wholesale when the per-agent block sets them. +func (a *AgentConfig) SandboxSubstrate(global AgentSandboxConfig) SandboxSubstrateConfig { + var g SandboxSubstrateConfig + if global.Substrate != nil { + g = *global.Substrate + } + if a == nil || a.Sandbox == nil || a.Sandbox.Substrate == nil { + return g + } + out := *a.Sandbox.Substrate + fallback := func(dst *string, src string) { + if strings.TrimSpace(*dst) == "" { + *dst = src + } + } + fallback(&out.Endpoint, g.Endpoint) + fallback(&out.Atespace, g.Atespace) + fallback(&out.TokenEnv, g.TokenEnv) + fallback(&out.TokenFile, g.TokenFile) + fallback(&out.CAFile, g.CAFile) + fallback(&out.RouterURL, g.RouterURL) + fallback(&out.RouterClientBundle, g.RouterClientBundle) + fallback(&out.RouterCAFile, g.RouterCAFile) + fallback(&out.Image, g.Image) + fallback(&out.PolicyProfile, g.PolicyProfile) + if out.TimeoutS == 0 { + out.TimeoutS = g.TimeoutS + } + if out.BridgePort == 0 { + out.BridgePort = g.BridgePort + } + if out.CPUMillicores == 0 { + out.CPUMillicores = g.CPUMillicores + } + if out.MemoryBytes == 0 { + out.MemoryBytes = g.MemoryBytes + } + if !out.InsecureTLS { + out.InsecureTLS = g.InsecureTLS + } + if !out.SnapshotOnIdle { + out.SnapshotOnIdle = g.SnapshotOnIdle + } + if len(out.BridgeCommand) == 0 { + out.BridgeCommand = g.BridgeCommand + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = g.AllowHosts + } + if len(out.WorkerSelector) == 0 { + out.WorkerSelector = g.WorkerSelector + } + return out +} diff --git a/src/crux/config/agent_config_substrate_test.go b/src/crux/config/agent_config_substrate_test.go new file mode 100644 index 0000000000..5b55bc2998 --- /dev/null +++ b/src/crux/config/agent_config_substrate_test.go @@ -0,0 +1,121 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxSubstrateFallsBackToGlobal(t *testing.T) { + var nilA *AgentConfig + if got := nilA.SandboxSubstrate(AgentSandboxConfig{}); !reflect.DeepEqual(got, SandboxSubstrateConfig{}) { + t.Fatalf("nil agent, nil global = %+v, want zero value", got) + } + + global := AgentSandboxConfig{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api.ate-system.svc:443", Atespace: "hive", TokenEnv: "ATE_TOKEN", TokenFile: "/tok", + CAFile: "/ca", InsecureTLS: true, TimeoutS: 900, RouterURL: "https://router:8444", + RouterClientBundle: "/bundle", RouterCAFile: "/router-ca", BridgePort: 9000, + Image: "img@sha256:abc", BridgeCommand: []string{"/bridge"}, CPUMillicores: 1000, MemoryBytes: 1 << 30, + WorkerSelector: map[string]string{"class": "gvisor"}, SnapshotOnIdle: true, + AllowHosts: []string{"hub.example"}, PolicyProfile: "strict", + }} + if got := nilA.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("nil agent = %+v, want global block", got) + } + noBlock := &AgentConfig{Sandbox: &AgentSandboxOverride{Image: "a-img"}} + if got := noBlock.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("agent without substrate block = %+v, want global block", got) + } + + // A per-agent block that names nothing but whitespace inherits every + // field from the global one. + empty := &AgentConfig{Sandbox: &AgentSandboxOverride{Substrate: &SandboxSubstrateConfig{Endpoint: " "}}} + if got := empty.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("blank per-agent block = %+v, want global block", got) + } +} + +func TestSandboxSubstratePerAgentFieldsWin(t *testing.T) { + global := AgentSandboxConfig{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api-global:443", Atespace: "global", TokenEnv: "GLOBAL_TOKEN", TokenFile: "/global-tok", + CAFile: "/global-ca", TimeoutS: 900, RouterURL: "https://router-global:8444", + RouterClientBundle: "/global-bundle", RouterCAFile: "/global-router-ca", BridgePort: 8477, + Image: "global@sha256:aaa", BridgeCommand: []string{"/global-bridge"}, CPUMillicores: 2000, MemoryBytes: 4 << 30, + WorkerSelector: map[string]string{"pool": "global"}, + AllowHosts: []string{"global.example", "hub.example"}, PolicyProfile: "global-policy", + }} + a := &AgentConfig{Sandbox: &AgentSandboxOverride{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api-agent:443", TimeoutS: 60, InsecureTLS: true, BridgePort: 9001, + Image: "agent@sha256:bbb", BridgeCommand: []string{"/agent-bridge"}, CPUMillicores: 500, MemoryBytes: 1 << 30, + WorkerSelector: map[string]string{"gpu": "a100"}, SnapshotOnIdle: true, + AllowHosts: []string{"models.example"}, + }}} + got := a.SandboxSubstrate(global) + + if got.Endpoint != "api-agent:443" || got.TimeoutS != 60 || !got.InsecureTLS || got.BridgePort != 9001 || got.Image != "agent@sha256:bbb" { + t.Fatalf("per-agent fields not kept: %+v", got) + } + if got.Atespace != "global" || got.TokenEnv != "GLOBAL_TOKEN" || got.TokenFile != "/global-tok" || got.CAFile != "/global-ca" || + got.RouterURL != "https://router-global:8444" || got.RouterClientBundle != "/global-bundle" || got.RouterCAFile != "/global-router-ca" || got.PolicyProfile != "global-policy" { + t.Fatalf("empty per-agent cluster facts not inherited: %+v", got) + } + if got.CPUMillicores != 500 || got.MemoryBytes != 1<<30 || !got.SnapshotOnIdle { + t.Fatalf("per-agent resources/snapshot not kept: %+v", got) + } + if !reflect.DeepEqual(got.AllowHosts, []string{"models.example"}) { + t.Fatalf("allow_hosts = %v, want per-agent list only", got.AllowHosts) + } + if !reflect.DeepEqual(got.BridgeCommand, []string{"/agent-bridge"}) { + t.Fatalf("bridge_command = %v, want per-agent list only", got.BridgeCommand) + } + if !reflect.DeepEqual(got.WorkerSelector, map[string]string{"gpu": "a100"}) { + t.Fatalf("worker_selector = %v, want per-agent map only", got.WorkerSelector) + } + + // Booleans: a global true cannot be switched off per-agent. + global.Substrate.InsecureTLS = true + global.Substrate.SnapshotOnIdle = true + a.Sandbox.Substrate.InsecureTLS = false + a.Sandbox.Substrate.SnapshotOnIdle = false + if merged := a.SandboxSubstrate(global); !merged.InsecureTLS || !merged.SnapshotOnIdle { + t.Fatal("per-agent false inherits global true") + } + + // The merge must not write the inherited values back into its inputs. + if global.Substrate.AllowHosts[0] != "global.example" || a.Sandbox.Substrate.Atespace != "" { + t.Fatal("merge mutated its inputs") + } +} + +func TestSubstratePreflightRegistry(t *testing.T) { + t.Cleanup(func() { SetSubstratePreflight(nil) }) + cfgs := []SandboxSubstrateConfig{{Endpoint: "api:443"}} + + SetSubstratePreflight(nil) + if got := SubstratePreflightWarnings(cfgs); got != nil { + t.Fatalf("unregistered preflight = %v, want nil", got) + } + + var seen []SandboxSubstrateConfig + SetSubstratePreflight(func(c []SandboxSubstrateConfig) []string { + seen = c + return []string{"control api unreachable"} + }) + if got := SubstratePreflightWarnings(nil); got != nil || seen != nil { + t.Fatalf("empty cfgs = %v (called=%v), want nil and no call", got, seen != nil) + } + if got := SubstratePreflightWarnings(cfgs); !reflect.DeepEqual(got, []string{"control api unreachable"}) || !reflect.DeepEqual(seen, cfgs) { + t.Fatalf("registered preflight = %v (seen %v), want its warnings for the given cfgs", got, seen) + } + + SetSubstratePreflight(nil) + if got := SubstratePreflightWarnings(cfgs); got != nil { + t.Fatalf("after unregister = %v, want nil", got) + } +} + +func TestValidSandboxRuntimesIncludesSubstrate(t *testing.T) { + if !ValidSandboxRuntimes[SandboxRuntimeSubstrate] { + t.Fatal("substrate is not a valid sandbox runtime") + } +} diff --git a/src/crux/go.mod b/src/crux/go.mod index ec82f74fff..f9e6c9f12a 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -16,6 +16,8 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/sys v0.49.0 + google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 ) @@ -50,6 +52,4 @@ require ( golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect - google.golang.org/grpc v1.85.0 // indirect - google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go new file mode 100644 index 0000000000..1ed56c0ee9 --- /dev/null +++ b/src/crux/sandbox/substrate.go @@ -0,0 +1,387 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Agent Substrate runtime (ADR-0026 phase 2; sibling of the OpenShell runtime +// in openshell.go). +// +// SubstrateLauncher implements Launcher on top of a Substrate control plane +// (ate-api-server): it registers a per-kick ActorTemplate from the kick spec, +// creates an Actor from it, attaches the egress policy, resumes the actor, +// drives the kick through the in-actor bridge (substratebridge) over the +// atenet-router CONNECT tunnel, and deletes — or, with snapshot_on_idle, +// suspends — the actor afterwards. Substrate owns the gVisor/microVM sandbox, +// the snapshot store and the egress gateway; hive's part is the spec, the +// policy and the same cleanup and timeout semantics the Job launcher has. +// +// Two points differ from the OpenShell and Job runtimes and are deliberate: +// +// - Substrate has no exec/attach RPC and cannot mount an existing PVC, so +// the workspace is a per-actor durableDir and hive copies it in and out +// through the bridge (push, run, pull). The local workspace hive opened +// the kick in is therefore still the source of truth after the kick. +// - Substrate refuses tag-only images; the sandbox image must be pinned by +// digest (…@sha256:…). Validate enforces it up front so the failure is a +// config error, not a CreateActorTemplate rejection mid-kick. + +const ( + // SubstrateDefaultEndpoint is the in-cluster Control API address the + // default ate-setup install exposes. + SubstrateDefaultEndpoint = "api.ate-system.svc:443" + // SubstrateDefaultRouterURL is the in-cluster atenet-router CONNECT + // listener hive reaches actors through. + SubstrateDefaultRouterURL = "https://atenet-router.ate-system.svc:8444" + // SubstrateDefaultAtespace is the atespace actors land in when none is + // configured. + SubstrateDefaultAtespace = "hive" + // SubstrateDefaultTokenFile is the projected ServiceAccount token + // (audience api.ate-system.svc) the Control API authenticates. + SubstrateDefaultTokenFile = "/var/run/secrets/ate/token" + // SubstrateDefaultCAFile is the servicedns ClusterTrustBundle projection + // the Control API serving certificate chains to. + SubstrateDefaultCAFile = "/var/run/secrets/ate/ca.crt" + // SubstrateDefaultRouterClientBundle is the projected podCertificate + // bundle atenet-router authenticates hive with. + SubstrateDefaultRouterClientBundle = "/run/router-client/credential-bundle.pem" + // SubstrateDefaultRouterCAFile is the servicedns ClusterTrustBundle the + // router's serving certificate chains to. + SubstrateDefaultRouterCAFile = "/run/router-client/router-ca.pem" + // SubstrateWorkspaceMount is where the actor sees the per-kick + // workspace, matching the other launchers. + SubstrateWorkspaceMount = substratebridge.DefaultWorkspace + // SubstrateDefaultCPUMillicores and SubstrateDefaultMemoryBytes are the + // template resource limits when the operator sets none; Substrate + // requires both. + SubstrateDefaultCPUMillicores = 2000 + SubstrateDefaultMemoryBytes = 4 << 30 + // SubstrateExitCodeUnknown mirrors OpenShellExitCodeUnknown. + SubstrateExitCodeUnknown = -1 + + // SubstratePolicyProfileLabel carries sandbox.substrate.policy_profile on + // the per-kick template's worker selector key space; Substrate has no + // annotations, so the profile travels as a template label hive and + // operators can filter on. + SubstratePolicyProfileLabel = "hive.dev/policy-profile" + + substrateNamePrefix = "hive-kick-" + substrateNameMax = 63 + substrateRequestWait = 30 * time.Second + substrateReadyPoll = 2 * time.Second + substrateEgressPolicy = "default" + substrateContainer = "agent" + substrateVolume = "workspace" + substrateHTTPSPort = 443 +) + +// SubstrateOptions is the launcher's configuration; crux/agent builds it +// from config.SandboxSubstrateConfig. +type SubstrateOptions struct { + Endpoint string + Atespace string + Token string + TokenFile string + CAFile string + InsecureTLS bool + Timeout time.Duration + RouterURL string + RouterClientBundle string + RouterCAFile string + BridgePort int + Image string + Command []string + CPUMillicores int64 + MemoryBytes int64 + WorkerSelector map[string]string + SnapshotOnIdle bool + AllowHosts []string + PolicyProfile string +} + +// Validate reports the first configuration error a kick would hit. +func (o SubstrateOptions) Validate() error { + if _, _, err := substrateAddress(o.endpoint()); err != nil { + return err + } + if _, err := substrateRouterAddress(o.routerURL()); err != nil { + return err + } + if o.Image != "" && !substrateImageDigest.MatchString(o.Image) { + return fmt.Errorf("substrate: image %q must be pinned by digest (…@sha256:<64 hex>); Substrate refuses tag-only images", o.Image) + } + if o.CPUMillicores < 0 || o.MemoryBytes < 0 { + return errors.New("substrate: cpu_millicores and memory_bytes must not be negative") + } + if o.BridgePort < 0 || o.BridgePort > 65535 { + return fmt.Errorf("substrate: bridge_port %d is out of range", o.BridgePort) + } + _, err := SubstrateEgressPolicy(o.AllowHosts) + return err +} + +func (o SubstrateOptions) endpoint() string { + if e := strings.TrimSpace(o.Endpoint); e != "" { + return e + } + return SubstrateDefaultEndpoint +} + +func (o SubstrateOptions) atespace() string { + if a := strings.TrimSpace(o.Atespace); a != "" { + return a + } + return SubstrateDefaultAtespace +} + +func (o SubstrateOptions) routerURL() string { + if r := strings.TrimSpace(o.RouterURL); r != "" { + return r + } + return SubstrateDefaultRouterURL +} + +func (o SubstrateOptions) bridgePort() int { + if o.BridgePort > 0 { + return o.BridgePort + } + return substratebridge.DefaultPort +} + +// substrateControl is the slice of the Control API the launcher uses; it +// exists so tests can drive the launcher with a fake and so an upstream +// RPC rename shows up here, not in every call site. +type substrateControl interface { + CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) + DeleteActorTemplate(ctx context.Context, ref *substrateapi.ObjectRef) error + CreateActor(ctx context.Context, actor *substrateapi.Actor) (*substrateapi.Actor, error) + GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) + ResumeActor(ctx context.Context, ref *substrateapi.ObjectRef) error + SuspendActor(ctx context.Context, ref *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error + DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error + CreateActorEgressPolicy(ctx context.Context, ref *substrateapi.ObjectRef, policy *substrateapi.EgressPolicy) error + ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) +} + +// substrateBridge is the launcher's view of the in-actor bridge. +type substrateBridge interface { + WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error + Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error + Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) + Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error +} + +type substrateClient interface { + Control() substrateControl + Bridge() substrateBridge + Close() error +} + +// SubstrateLauncher runs sandboxed kicks as Substrate actors. +type SubstrateLauncher struct { + Options SubstrateOptions + + // newClient overrides the client constructor in tests. + newClient func(SubstrateOptions) (substrateClient, error) + // now overrides the clock in tests. + now func() time.Time +} + +func (l *SubstrateLauncher) client() (substrateClient, error) { + if l.newClient != nil { + return l.newClient(l.Options) + } + return newSubstrateGRPCClient(l.Options) +} + +// Run implements Launcher. +func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, error) { + res := Result{ExitCode: SubstrateExitCodeUnknown} + if spec.ClineProvider != nil { + _ = spec.ClineProvider.Close() + return res, errors.New("substrate: scoped Cline provider files are Podman-only; the Substrate runtime takes no credential files") + } + if err := l.Options.Validate(); err != nil { + return res, err + } + // Validate just proved the hosts parse, so this cannot fail; the policy + // is built before anything is created so a kick never has a half-built + // actor to clean up on a config error. + policy, err := SubstrateEgressPolicy(l.Options.AllowHosts) + if err != nil { + return res, err + } + image := strings.TrimSpace(l.Options.Image) + if image == "" { + image = strings.TrimSpace(spec.Image) + } + if image == "" { + return res, errors.New("substrate: image is required") + } + if !substrateImageDigest.MatchString(image) { + return res, fmt.Errorf("substrate: image %q must be pinned by digest (…@sha256:<64 hex>); set sandbox.substrate.image to the digest-pinned sandbox image", image) + } + if strings.TrimSpace(spec.Workspace) == "" { + return res, errors.New("substrate: workspace is required") + } + if len(spec.Command) == 0 { + return res, errors.New("substrate: command is required") + } + if l.Options.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, l.Options.Timeout) + defer cancel() + } + + client, err := l.client() + if err != nil { + return res, err + } + defer func() { _ = client.Close() }() + control, bridge := client.Control(), client.Bridge() + + now := time.Now + if l.now != nil { + now = l.now + } + name := substrateActorName(spec.Name, now()) + ref := &substrateapi.ObjectRef{Atespace: l.Options.atespace(), Name: name} + + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests; cleanup falls back to a fresh, request-bounded + // context so a finished actor is still removed. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), substrateRequestWait) + cancels = append(cancels, cancel) + return fresh + } + + if _, err := control.CreateActorTemplate(ctx, l.actorTemplate(ref, image)); err != nil { + return res, fmt.Errorf("substrate: creating actor template %s: %w", name, err) + } + cleanupTemplate := func() { _ = control.DeleteActorTemplate(live(), ref) } + + if _, err := control.CreateActor(ctx, &substrateapi.Actor{ + Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: name}, + ActorTemplate: ref, + WorkerSelector: substrateSelector(l.Options.WorkerSelector), + }); err != nil { + cleanupTemplate() + return res, fmt.Errorf("substrate: creating actor %s: %w", name, err) + } + // Delete is the default: a kick is a one-shot and keeping a snapshot + // around costs object storage. snapshot_on_idle keeps a resumable + // snapshot instead, so the next kick's warm state (caches, builds) is + // a resume rather than a cold boot. + cleanupActor := func() { + if l.Options.SnapshotOnIdle && ctx.Err() == nil { + if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { + cleanupTemplate() + return + } + } + _ = control.DeleteActor(live(), ref) + cleanupTemplate() + } + + if spec.NetworkNone { + policy.Rules = nil + } + if err := control.CreateActorEgressPolicy(ctx, ref, policy); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: setting egress policy on actor %s: %w", name, err) + } + if err := control.ResumeActor(ctx, ref); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: resuming actor %s: %w", name, err) + } + if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll); err != nil { + cleanupActor() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("substrate: actor %s never became running: %w", name, err) + } + if err := bridge.WaitReady(ctx, ref, substrateReadyPoll); err != nil { + cleanupActor() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("substrate: bridge in actor %s never answered: %w", name, err) + } + if err := bridge.Push(ctx, ref, spec.Workspace); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: pushing workspace into actor %s: %w", name, err) + } + workDir := spec.WorkDir + if workDir == "" { + workDir = SubstrateWorkspaceMount + } + out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}) + if runErr == nil { + res.Stdout, res.Stderr, res.ExitCode = out.Stdout, out.Stderr, out.ExitCode + if out.Error != "" { + runErr = errors.New(out.Error) + } + } + var pullErr error + if ctx.Err() == nil { + pullErr = bridge.Pull(ctx, ref, spec.Workspace) + } + cleanupActor() + if ctx.Err() != nil { + // Kick timeout or cancel: the actor has been removed above so the + // workload does not keep running on the cluster. + return res, ctx.Err() + } + if runErr != nil { + return res, fmt.Errorf("substrate: running kick in actor %s: %w", name, runErr) + } + if pullErr != nil { + return res, fmt.Errorf("substrate: pulling workspace out of actor %s: %w", name, pullErr) + } + return res, nil +} + +// substrateWaitRunning polls GetActor until the actor is RUNNING, or fails +// when it crashes. A deleted or suspended actor during the wait is a failure +// too: nothing hive did asked for it. +func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration) error { + ticker := time.NewTicker(poll) + defer ticker.Stop() + for { + actor, err := control.GetActor(ctx, ref) + if err != nil { + return err + } + switch state := actor.GetStatus().GetState(); state { + case substrateapi.ActorState_ACTOR_STATE_RUNNING: + return nil + case substrateapi.ActorState_ACTOR_STATE_CRASHED: + return fmt.Errorf("actor crashed: %s", actor.GetStatus().GetCrash().GetMessage()) + case substrateapi.ActorState_ACTOR_STATE_DELETING, substrateapi.ActorState_ACTOR_STATE_SUSPENDING: + return fmt.Errorf("actor entered %s while being resumed", state) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} diff --git a/src/crux/sandbox/substrate_bridge.go b/src/crux/sandbox/substrate_bridge.go new file mode 100644 index 0000000000..77f3ca4a16 --- /dev/null +++ b/src/crux/sandbox/substrate_bridge.go @@ -0,0 +1,107 @@ +package sandbox + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// routerBridge's substrateBridge implementation: the four verbs the launcher +// drives a kick with, each one HTTP exchange over its own CONNECT tunnel. + +// WaitReady polls the bridge's health endpoint until it answers. The actor +// is RUNNING by now, but the bridge process inside still has to come up. +func (b *routerBridge) WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error { + ticker := time.NewTicker(poll) + defer ticker.Stop() + var last error + for { + resp, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathHealth, nil) + if err == nil { + resp.Body.Close() + return nil + } + last = err + select { + case <-ctx.Done(): + return fmt.Errorf("%w (last: %v)", ctx.Err(), last) + case <-ticker.C: + } + } +} + +// Push replaces the actor's workspace with the local one. +func (b *routerBridge) Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error { + var buf bytes.Buffer + if err := substratebridge.Pack(workspace, &buf); err != nil { + return fmt.Errorf("packing %s: %w", workspace, err) + } + resp, err := b.do(ctx, ref, http.MethodPut, substratebridge.PathWorkspace, &buf) + if err != nil { + return err + } + return resp.Body.Close() +} + +// Run executes the kick command in the actor and waits for it to exit. +func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { + var out substratebridge.RunResponse + body, err := json.Marshal(req) + if err != nil { + return out, err + } + resp, err := b.do(ctx, ref, http.MethodPost, substratebridge.PathRun, bytes.NewReader(body)) + if err != nil { + return out, err + } + defer resp.Body.Close() + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return out, fmt.Errorf("decoding run response: %w", err) + } + return out, nil +} + +// Pull replaces the local workspace with the actor's, so the commits the +// agent made are what hive's post-kick push broker sees. The new tree is +// unpacked beside the old one and swapped in, so a failed transfer leaves +// the original workspace untouched. +func (b *routerBridge) Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error { + resp, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathWorkspace, nil) + if err != nil { + return err + } + defer resp.Body.Close() + return substrateReplaceDir(resp.Body, workspace) +} + +func substrateReplaceDir(stream io.Reader, workspace string) error { + parent := filepath.Dir(filepath.Clean(workspace)) + staging, err := os.MkdirTemp(parent, ".substrate-pull-*") + if err != nil { + return err + } + if err := substratebridge.Unpack(stream, staging); err != nil { + _ = os.RemoveAll(staging) + return fmt.Errorf("unpacking into %s: %w", staging, err) + } + old := staging + ".old" + if err := os.Rename(workspace, old); err != nil { + _ = os.RemoveAll(staging) + return err + } + if err := os.Rename(staging, workspace); err != nil { + _ = os.Rename(old, workspace) + _ = os.RemoveAll(staging) + return err + } + return os.RemoveAll(old) +} diff --git a/src/crux/sandbox/substrate_client.go b/src/crux/sandbox/substrate_client.go new file mode 100644 index 0000000000..2e72b7e4a5 --- /dev/null +++ b/src/crux/sandbox/substrate_client.go @@ -0,0 +1,292 @@ +package sandbox + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Real Substrate client: the Control API over gRPC plus the in-actor bridge +// over an atenet-router CONNECT tunnel. Constructed lazily by +// SubstrateLauncher.client() so unit tests never dial anything. + +// substrateRouterHeader selects the target actor on the router's CONNECT +// request (api-guide "Workload Connectivity"). +const substrateRouterHeader = "ate-target-actor" + +type grpcSubstrateClient struct { + conn *grpc.ClientConn + control *grpcControl + bridge *routerBridge +} + +func (c *grpcSubstrateClient) Control() substrateControl { return c.control } +func (c *grpcSubstrateClient) Bridge() substrateBridge { return c.bridge } +func (c *grpcSubstrateClient) Close() error { return c.conn.Close() } + +func newSubstrateGRPCClient(o SubstrateOptions) (substrateClient, error) { + addr, useTLS, err := substrateAddress(o.endpoint()) + if err != nil { + return nil, err + } + creds := insecure.NewCredentials() + if useTLS { + cfg, err := substrateControlTLS(o) + if err != nil { + return nil, err + } + creds = credentials.NewTLS(cfg) + } + opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + if o.Token != "" || o.TokenFile != "" || useTLS { + opts = append(opts, grpc.WithPerRPCCredentials(substrateBearer{token: o.Token, file: o.tokenFile(), secure: useTLS})) + } + conn, err := grpc.NewClient(addr, opts...) + if err != nil { + return nil, fmt.Errorf("substrate: connecting to control api %s: %w", addr, err) + } + bridge, err := newRouterBridge(o) + if err != nil { + _ = conn.Close() + return nil, err + } + return &grpcSubstrateClient{conn: conn, control: &grpcControl{c: substrateapi.NewControlClient(conn)}, bridge: bridge}, nil +} + +func (o SubstrateOptions) tokenFile() string { + if f := strings.TrimSpace(o.TokenFile); f != "" { + return f + } + return SubstrateDefaultTokenFile +} + +// substrateControlTLS verifies the Control API against ca_file (the +// servicedns ClusterTrustBundle projection) when present, the system roots +// otherwise, or nothing with insecure_tls. +func substrateControlTLS(o SubstrateOptions) (*tls.Config, error) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.InsecureTLS} //nolint:gosec // operator opt-in + if o.InsecureTLS { + return cfg, nil + } + caFile := strings.TrimSpace(o.CAFile) + if caFile == "" { + if _, err := os.Stat(SubstrateDefaultCAFile); err == nil { + caFile = SubstrateDefaultCAFile + } + } + if caFile != "" { + pool, err := substrateCertPool(caFile) + if err != nil { + return nil, fmt.Errorf("substrate: ca_file: %w", err) + } + cfg.RootCAs = pool + } + return cfg, nil +} + +func substrateCertPool(path string) (*x509.CertPool, error) { + pem, err := os.ReadFile(path) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("%s holds no PEM certificates", path) + } + return pool, nil +} + +// substrateBearer sends the projected ServiceAccount token as a bearer +// token, re-reading the file per RPC so kubelet rotation is picked up. +type substrateBearer struct { + token string + file string + secure bool +} + +func (b substrateBearer) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + tok := b.token + if tok == "" { + raw, err := os.ReadFile(b.file) + if err != nil { + return nil, fmt.Errorf("substrate: reading token file %s: %w", b.file, err) + } + tok = strings.TrimSpace(string(raw)) + } + if tok == "" { + return nil, errors.New("substrate: control api token is empty") + } + return map[string]string{"authorization": "Bearer " + tok}, nil +} + +func (b substrateBearer) RequireTransportSecurity() bool { return b.secure } + +// grpcControl adapts the generated ControlClient to substrateControl. +type grpcControl struct{ c substrateapi.ControlClient } + +func (g *grpcControl) CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) { + return g.c.CreateActorTemplate(ctx, &substrateapi.CreateActorTemplateRequest{ActorTemplate: tpl}) +} + +func (g *grpcControl) DeleteActorTemplate(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.DeleteActorTemplate(ctx, &substrateapi.DeleteActorTemplateRequest{ActorTemplate: ref}) + return err +} + +func (g *grpcControl) CreateActor(ctx context.Context, actor *substrateapi.Actor) (*substrateapi.Actor, error) { + return g.c.CreateActor(ctx, &substrateapi.CreateActorRequest{Actor: actor}) +} + +func (g *grpcControl) GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) { + return g.c.GetActor(ctx, &substrateapi.GetActorRequest{Actor: ref}) +} + +func (g *grpcControl) ResumeActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.ResumeActor(ctx, &substrateapi.ResumeActorRequest{Actor: ref}) + return err +} + +func (g *grpcControl) SuspendActor(ctx context.Context, ref *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error { + _, err := g.c.SuspendActor(ctx, &substrateapi.SuspendActorRequest{Actor: ref, Options: &substrateapi.SuspendOptions{Mode: mode}}) + return err +} + +func (g *grpcControl) DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.DeleteActor(ctx, &substrateapi.DeleteActorRequest{Actor: ref}) + return err +} + +func (g *grpcControl) CreateActorEgressPolicy(ctx context.Context, ref *substrateapi.ObjectRef, policy *substrateapi.EgressPolicy) error { + _, err := g.c.CreateActorEgressPolicy(ctx, &substrateapi.CreateActorEgressPolicyRequest{Actor: ref, EgressPolicy: policy}) + return err +} + +func (g *grpcControl) ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) { + var out []*substrateapi.Worker + req := &substrateapi.ListWorkersRequest{Options: &substrateapi.ListOptions{}} + for { + resp, err := g.c.ListWorkers(ctx, req) + if err != nil { + return nil, err + } + out = append(out, resp.GetWorkers()...) + if resp.GetNextPageToken() == "" { + return out, nil + } + req.Options.PageToken = resp.GetNextPageToken() + } +} + +// routerBridge reaches the in-actor bridge through atenet-router: an mTLS +// CONNECT to the router with the actor named in ate-target-actor, then plain +// HTTP to the bridge over the tunnel. +type routerBridge struct { + routerAddr string + tlsConfig *tls.Config + port int + dial func(ctx context.Context, network, addr string) (net.Conn, error) +} + +func newRouterBridge(o SubstrateOptions) (*routerBridge, error) { + addr, err := substrateRouterAddress(o.routerURL()) + if err != nil { + return nil, err + } + host, _, _ := net.SplitHostPort(addr) + cfg := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: host, InsecureSkipVerify: o.InsecureTLS} //nolint:gosec // operator opt-in + bundle := o.RouterClientBundle + if bundle == "" { + bundle = SubstrateDefaultRouterClientBundle + } + cert, err := tls.LoadX509KeyPair(bundle, bundle) + if err != nil { + return nil, fmt.Errorf("substrate: router_client_bundle %s: %w — atenet-router needs a podCertificate projection for hive (see deploy/substrate.md)", bundle, err) + } + cfg.Certificates = []tls.Certificate{cert} + if !o.InsecureTLS { + caFile := o.RouterCAFile + if caFile == "" { + caFile = SubstrateDefaultRouterCAFile + } + pool, err := substrateCertPool(caFile) + if err != nil { + return nil, fmt.Errorf("substrate: router_ca_file: %w", err) + } + cfg.RootCAs = pool + } + d := &net.Dialer{Timeout: substrateRequestWait} + return &routerBridge{routerAddr: addr, tlsConfig: cfg, port: o.bridgePort(), dial: d.DialContext}, nil +} + +// tunnel opens one CONNECT tunnel to the actor's bridge port. +func (b *routerBridge) tunnel(ctx context.Context, ref *substrateapi.ObjectRef) (net.Conn, error) { + raw, err := b.dial(ctx, "tcp", b.routerAddr) + if err != nil { + return nil, err + } + conn := tls.Client(raw, b.tlsConfig) + if err := conn.HandshakeContext(ctx); err != nil { + raw.Close() + return nil, fmt.Errorf("router tls handshake: %w", err) + } + target := net.JoinHostPort("actor", fmt.Sprint(b.port)) + req := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: http.Header{}} + req.Header.Set(substrateRouterHeader, ref.GetAtespace()+"/"+ref.GetName()) + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } + if err := req.Write(conn); err != nil { + conn.Close() + return nil, fmt.Errorf("router connect: %w", err) + } + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + conn.Close() + return nil, fmt.Errorf("router connect: %w", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + conn.Close() + return nil, fmt.Errorf("router refused connect: %s", resp.Status) + } + _ = conn.SetDeadline(time.Time{}) + return conn, nil +} + +func (b *routerBridge) do(ctx context.Context, ref *substrateapi.ObjectRef, method, path string, body io.Reader) (*http.Response, error) { + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return b.tunnel(ctx, ref) }, + DisableKeepAlives: true, + } + defer transport.CloseIdleConnections() + req, err := http.NewRequestWithContext(ctx, method, "http://actor"+path, body) + if err != nil { + return nil, err + } + resp, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode/100 != 2 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + return nil, fmt.Errorf("bridge %s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(msg))) + } + return resp, nil +} diff --git a/src/crux/sandbox/substrate_client_test.go b/src/crux/sandbox/substrate_client_test.go new file mode 100644 index 0000000000..fa9f0c8ea4 --- /dev/null +++ b/src/crux/sandbox/substrate_client_test.go @@ -0,0 +1,681 @@ +package sandbox + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "io" + "log" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Real-client tests: the gRPC adapter against an in-memory Control server, +// TLS/credential plumbing against files in a temp dir, and the router bridge +// against a TLS server that speaks CONNECT and then the bridge protocol. + +// selfSignedBundle writes a PEM file holding a self-signed cert and its key +// (the shape of a podCertificate credential bundle) plus a CA-only file. +func selfSignedBundle(t *testing.T, dir string) (bundle, caOnly string, cert tls.Certificate) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "hive-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + bundle = filepath.Join(dir, "credential-bundle.pem") + caOnly = filepath.Join(dir, "router-ca.pem") + if err := os.WriteFile(bundle, append(certPEM, keyPEM...), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(caOnly, certPEM, 0o600); err != nil { + t.Fatal(err) + } + cert, err = tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + t.Fatal(err) + } + return bundle, caOnly, cert +} + +func TestNewSubstrateGRPCClientConstructs(t *testing.T) { + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + + // TLS endpoint verified against ca_file, router verified against its CA. + c, err := newSubstrateGRPCClient(SubstrateOptions{ + Endpoint: "api.ate-system.svc:443", CAFile: ca, TokenFile: filepath.Join(dir, "token"), + RouterClientBundle: bundle, RouterCAFile: ca, + }) + if err != nil { + t.Fatalf("tls client: %v", err) + } + if c.Control() == nil || c.Bridge() == nil { + t.Error("nil control or bridge") + } + if err := c.Close(); err != nil { + t.Errorf("Close: %v", err) + } + + // Plaintext endpoint with an env token and insecure router. + c, err = newSubstrateGRPCClient(SubstrateOptions{Endpoint: "http://localhost:8080", Token: "t", InsecureTLS: true, RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("plaintext client: %v", err) + } + _ = c.Close() + + // Plaintext without any token still constructs (no per-RPC creds). + c, err = newSubstrateGRPCClient(SubstrateOptions{Endpoint: "http://localhost:8080", InsecureTLS: true, RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("plaintext no-token client: %v", err) + } + _ = c.Close() + + // Insecure TLS skips the CA file entirely. + c, err = newSubstrateGRPCClient(SubstrateOptions{InsecureTLS: true, CAFile: "/nonexistent", RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("insecure client: %v", err) + } + _ = c.Close() +} + +func TestNewSubstrateGRPCClientErrors(t *testing.T) { + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + noPEM := filepath.Join(dir, "empty.pem") + if err := os.WriteFile(noPEM, []byte("not a cert"), 0o600); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + o SubstrateOptions + want string + }{ + {"endpoint", SubstrateOptions{Endpoint: "nope"}, "endpoint"}, + {"ca missing", SubstrateOptions{CAFile: "/nonexistent/ca.crt"}, "ca_file"}, + {"ca not pem", SubstrateOptions{CAFile: noPEM}, "no PEM certificates"}, + {"router url", SubstrateOptions{CAFile: ca, RouterURL: "http://r"}, "router_url"}, + {"router bundle", SubstrateOptions{CAFile: ca, RouterClientBundle: "/nonexistent/b.pem"}, "router_client_bundle"}, + {"router ca", SubstrateOptions{CAFile: ca, RouterClientBundle: bundle, RouterCAFile: "/nonexistent/ca.pem"}, "router_ca_file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := newSubstrateGRPCClient(tc.o) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + }) + } +} + +func TestSubstrateControlTLSDefaults(t *testing.T) { + cfg, err := substrateControlTLS(SubstrateOptions{}) + if err != nil || cfg.RootCAs != nil || cfg.InsecureSkipVerify { + t.Errorf("default tls = %+v, %v (system roots expected when the projected CA is absent)", cfg, err) + } + _, ca, _ := selfSignedBundle(t, t.TempDir()) + cfg, err = substrateControlTLS(SubstrateOptions{CAFile: ca}) + if err != nil || cfg.RootCAs == nil { + t.Errorf("ca_file tls = %+v, %v", cfg, err) + } +} + +func TestSubstrateBearer(t *testing.T) { + ctx := context.Background() + md, err := substrateBearer{token: "abc", secure: true}.GetRequestMetadata(ctx) + if err != nil || md["authorization"] != "Bearer abc" { + t.Errorf("static token: %v %v", md, err) + } + dir := t.TempDir() + f := filepath.Join(dir, "token") + if err := os.WriteFile(f, []byte(" from-file\n"), 0o600); err != nil { + t.Fatal(err) + } + md, err = substrateBearer{file: f}.GetRequestMetadata(ctx) + if err != nil || md["authorization"] != "Bearer from-file" { + t.Errorf("file token: %v %v", md, err) + } + if _, err := (substrateBearer{file: filepath.Join(dir, "missing")}).GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), "reading token file") { + t.Errorf("missing file: %v", err) + } + if err := os.WriteFile(f, []byte(" \n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := (substrateBearer{file: f}).GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), "token is empty") { + t.Errorf("empty file: %v", err) + } + if !(substrateBearer{secure: true}).RequireTransportSecurity() || (substrateBearer{}).RequireTransportSecurity() { + t.Error("RequireTransportSecurity must follow the TLS choice") + } +} + +// controlServer is the in-memory Control API the adapter is tested against. +type controlServer struct { + substrateapi.UnimplementedControlServer + fail bool + pages int +} + +func (s *controlServer) err() error { + if s.fail { + return status.Error(codes.Internal, "injected") + } + return nil +} + +func (s *controlServer) CreateActorTemplate(_ context.Context, r *substrateapi.CreateActorTemplateRequest) (*substrateapi.ActorTemplate, error) { + return r.GetActorTemplate(), s.err() +} +func (s *controlServer) DeleteActorTemplate(context.Context, *substrateapi.DeleteActorTemplateRequest) (*substrateapi.ActorTemplate, error) { + return &substrateapi.ActorTemplate{}, s.err() +} +func (s *controlServer) CreateActor(_ context.Context, r *substrateapi.CreateActorRequest) (*substrateapi.Actor, error) { + return r.GetActor(), s.err() +} +func (s *controlServer) GetActor(_ context.Context, r *substrateapi.GetActorRequest) (*substrateapi.Actor, error) { + return &substrateapi.Actor{Metadata: &substrateapi.ResourceMetadata{Name: r.GetActor().GetName()}}, s.err() +} +func (s *controlServer) ResumeActor(context.Context, *substrateapi.ResumeActorRequest) (*substrateapi.ResumeActorResponse, error) { + return &substrateapi.ResumeActorResponse{}, s.err() +} +func (s *controlServer) SuspendActor(_ context.Context, r *substrateapi.SuspendActorRequest) (*substrateapi.SuspendActorResponse, error) { + if r.GetOptions().GetMode() != substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { + return nil, status.Error(codes.InvalidArgument, "mode not forwarded") + } + return &substrateapi.SuspendActorResponse{}, s.err() +} +func (s *controlServer) DeleteActor(context.Context, *substrateapi.DeleteActorRequest) (*substrateapi.Actor, error) { + return &substrateapi.Actor{}, s.err() +} +func (s *controlServer) CreateActorEgressPolicy(_ context.Context, r *substrateapi.CreateActorEgressPolicyRequest) (*substrateapi.EgressPolicy, error) { + return r.GetEgressPolicy(), s.err() +} +func (s *controlServer) ListWorkers(_ context.Context, r *substrateapi.ListWorkersRequest) (*substrateapi.ListWorkersResponse, error) { + if err := s.err(); err != nil { + return nil, err + } + s.pages++ + if r.GetOptions().GetPageToken() == "" { + return &substrateapi.ListWorkersResponse{Workers: []*substrateapi.Worker{{Metadata: &substrateapi.ResourceMetadata{Name: "w1"}}}, NextPageToken: "next"}, nil + } + return &substrateapi.ListWorkersResponse{Workers: []*substrateapi.Worker{{Metadata: &substrateapi.ResourceMetadata{Name: "w2"}}}}, nil +} + +func newBufconnControl(t *testing.T, srv substrateapi.ControlServer) *grpcControl { + t.Helper() + ln := bufconn.Listen(1 << 20) + gs := grpc.NewServer() + substrateapi.RegisterControlServer(gs, srv) + go func() { _ = gs.Serve(ln) }() + t.Cleanup(gs.Stop) + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return ln.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &grpcControl{c: substrateapi.NewControlClient(conn)} +} + +func TestGRPCControlAdapter(t *testing.T) { + srv := &controlServer{} + g := newBufconnControl(t, srv) + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a1"} + if tpl, err := g.CreateActorTemplate(ctx, &substrateapi.ActorTemplate{Metadata: &substrateapi.ResourceMetadata{Name: "t"}}); err != nil || tpl.GetMetadata().GetName() != "t" { + t.Errorf("CreateActorTemplate: %v %v", tpl, err) + } + if err := g.DeleteActorTemplate(ctx, ref); err != nil { + t.Errorf("DeleteActorTemplate: %v", err) + } + if a, err := g.CreateActor(ctx, &substrateapi.Actor{Metadata: &substrateapi.ResourceMetadata{Name: "a1"}}); err != nil || a.GetMetadata().GetName() != "a1" { + t.Errorf("CreateActor: %v %v", a, err) + } + if a, err := g.GetActor(ctx, ref); err != nil || a.GetMetadata().GetName() != "a1" { + t.Errorf("GetActor: %v %v", a, err) + } + if err := g.ResumeActor(ctx, ref); err != nil { + t.Errorf("ResumeActor: %v", err) + } + if err := g.SuspendActor(ctx, ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err != nil { + t.Errorf("SuspendActor: %v", err) + } + if err := g.DeleteActor(ctx, ref); err != nil { + t.Errorf("DeleteActor: %v", err) + } + if err := g.CreateActorEgressPolicy(ctx, ref, &substrateapi.EgressPolicy{}); err != nil { + t.Errorf("CreateActorEgressPolicy: %v", err) + } + workers, err := g.ListWorkers(ctx) + if err != nil || len(workers) != 2 || srv.pages != 2 { + t.Errorf("ListWorkers: %d workers, %d pages, %v", len(workers), srv.pages, err) + } + + srv.fail = true + if _, err := g.ListWorkers(ctx); status.Code(err) != codes.Internal { + t.Errorf("ListWorkers error not propagated: %v", err) + } + if _, err := g.CreateActorTemplate(ctx, nil); status.Code(err) != codes.Internal { + t.Errorf("CreateActorTemplate error not propagated: %v", err) + } +} + +// connectRouter is a TLS server that accepts CONNECT like atenet-router and +// then answers one bridge request over the tunnel. +type connectRouter struct { + t *testing.T + handler http.Handler + refuse bool + lastHdr string + lastHost string +} + +func (r *connectRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodConnect { + http.Error(w, "not connect", http.StatusMethodNotAllowed) + return + } + r.lastHdr = req.Header.Get(substrateRouterHeader) + r.lastHost = req.Host + if r.refuse { + http.Error(w, "spiffe id not allowed", http.StatusForbidden) + return + } + hj, ok := w.(http.Hijacker) + if !ok { + r.t.Fatal("no hijacker") + } + conn, rw, err := hj.Hijack() + if err != nil { + r.t.Fatal(err) + } + defer conn.Close() + _, _ = rw.WriteString("HTTP/1.1 200 Connection established\r\n\r\n") + _ = rw.Flush() + inner, err := http.ReadRequest(bufio.NewReader(conn)) + if err != nil { + return + } + rec := httptest.NewRecorder() + r.handler.ServeHTTP(rec, inner) + resp := rec.Result() + resp.ContentLength = int64(rec.Body.Len()) + _ = resp.Write(conn) +} + +func newConnectRouter(t *testing.T, h http.Handler) (*connectRouter, *routerBridge) { + t.Helper() + cr := &connectRouter{t: t, handler: h} + srv := httptest.NewUnstartedServer(cr) + // The untrusted-certificate case below makes the server log a handshake + // error; that is the expected outcome, not noise worth printing. + srv.Config.ErrorLog = log.New(io.Discard, "", 0) + srv.StartTLS() + t.Cleanup(srv.Close) + addr := strings.TrimPrefix(srv.URL, "https://") + b := &routerBridge{ + routerAddr: addr, + tlsConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, //nolint:gosec // test server + port: substratebridge.DefaultPort, + dial: (&net.Dialer{}).DialContext, + } + return cr, b +} + +func TestRouterBridgeDrivesTheBridgeProtocol(t *testing.T) { + actorWS := filepath.Join(t.TempDir(), "actor-ws") + cr, b := newConnectRouter(t, &substratebridge.Handler{Workspace: actorWS}) + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "hive-kick-deps-1"} + + if err := b.WaitReady(ctx, ref, time.Millisecond); err != nil { + t.Fatalf("WaitReady: %v", err) + } + if cr.lastHdr != "hive/hive-kick-deps-1" || cr.lastHost != "actor:8477" { + t.Errorf("CONNECT header=%q host=%q", cr.lastHdr, cr.lastHost) + } + + local := filepath.Join(t.TempDir(), "ws") + if err := os.MkdirAll(filepath.Join(local, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(local, "sub", "file.txt"), []byte("before"), 0o644); err != nil { + t.Fatal(err) + } + if err := b.Push(ctx, ref, local); err != nil { + t.Fatalf("Push: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(actorWS, "sub", "file.txt")); string(got) != "before" { + t.Fatalf("actor workspace = %q", got) + } + out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}) + if err != nil || out.ExitCode != 4 || !strings.Contains(out.Stdout, "ran") { + t.Fatalf("Run: %+v %v", out, err) + } + if err := b.Pull(ctx, ref, local); err != nil { + t.Fatalf("Pull: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(local, "sub", "file.txt")); strings.TrimSpace(string(got)) != "after" { + t.Errorf("local workspace after pull = %q", got) + } + if entries, _ := os.ReadDir(filepath.Dir(local)); len(entries) != 1 { + t.Errorf("staging dirs left behind: %v", entries) + } +} + +func TestRouterBridgeErrors(t *testing.T) { + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} + + // Router refuses the CONNECT. + cr, b := newConnectRouter(t, http.NotFoundHandler()) + cr.refuse = true + if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "router refused connect") { + t.Errorf("refused connect: %v", err) + } + + // Bridge answers non-2xx. + _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unpack failed", http.StatusBadRequest) + })) + if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "unpack failed") { + t.Errorf("non-2xx: %v", err) + } + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil { + t.Error("Run swallowed a non-2xx") + } + if err := b.Pull(ctx, ref, t.TempDir()); err == nil { + t.Error("Pull swallowed a non-2xx") + } + + // Bridge answers 200 with garbage JSON / garbage tar. + _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("{not json")) + })) + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil || !strings.Contains(err.Error(), "decoding run response") { + t.Errorf("bad json: %v", err) + } + ws := filepath.Join(t.TempDir(), "ws") + if err := os.Mkdir(ws, 0o755); err != nil { + t.Fatal(err) + } + if err := b.Pull(ctx, ref, ws); err == nil || !strings.Contains(err.Error(), "unpacking") { + t.Errorf("bad tar: %v", err) + } + + // Push of a workspace that does not exist fails before any I/O. + if err := b.Push(ctx, ref, filepath.Join(t.TempDir(), "missing")); err == nil || !strings.Contains(err.Error(), "packing") { + t.Errorf("missing workspace: %v", err) + } + + // TLS to a server whose certificate is not trusted. + _, strict := newConnectRouter(t, http.NotFoundHandler()) + strict.tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: x509.NewCertPool(), ServerName: "localhost"} + if _, err := strict.tunnel(ctx, ref); err == nil || !strings.Contains(err.Error(), "tls handshake") { + t.Errorf("untrusted server: %v", err) + } + + // Dial failure. + b.dial = func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("no route") } + if _, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathHealth, nil); err == nil || !strings.Contains(err.Error(), "no route") { + t.Errorf("dial failure: %v", err) + } + + // Server closes the tunnel without a CONNECT response. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + _, _, serverCert := selfSignedBundle(t, t.TempDir()) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + tc := tls.Server(c, &tls.Config{Certificates: []tls.Certificate{serverCert}, MinVersion: tls.VersionTLS12}) + _ = tc.Handshake() + _ = tc.Close() + } + }() + closing := &routerBridge{routerAddr: ln.Addr().String(), tlsConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, port: 1, dial: (&net.Dialer{}).DialContext} //nolint:gosec // test server + if _, err := closing.tunnel(ctx, ref); err == nil || !strings.Contains(err.Error(), "router connect") { + t.Errorf("closed tunnel: %v", err) + } +} + +func TestRouterBridgeWaitReadyGivesUpOnContext(t *testing.T) { + // The bridge answers 503 and the kick is cancelled right after that + // first probe, so WaitReady must stop without waiting for the ticker. + ctx, cancel := context.WithCancel(context.Background()) + _, b := newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + cancel() + http.Error(w, "starting", http.StatusServiceUnavailable) + })) + err := b.WaitReady(ctx, &substrateapi.ObjectRef{Atespace: "hive", Name: "a"}, time.Hour) + if !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "last:") { + t.Fatalf("err = %v, want context.Canceled wrapping the last probe error", err) + } +} + +func TestRouterBridgeRequestContextCancelled(t *testing.T) { + _, b := newConnectRouter(t, http.NotFoundHandler()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := b.do(ctx, &substrateapi.ObjectRef{}, http.MethodGet, substratebridge.PathHealth, nil); err == nil { + t.Error("do with a cancelled context succeeded") + } + if _, err := b.do(context.Background(), &substrateapi.ObjectRef{}, "bad method", "/x", nil); err == nil { + t.Error("do with an invalid method succeeded") + } +} + +func TestNewRouterBridgeLoadsCredentials(t *testing.T) { + bundle, ca, _ := selfSignedBundle(t, t.TempDir()) + b, err := newRouterBridge(SubstrateOptions{RouterClientBundle: bundle, RouterCAFile: ca, BridgePort: 9001, RouterURL: "https://router.example:8444"}) + if err != nil { + t.Fatalf("newRouterBridge: %v", err) + } + if b.routerAddr != "router.example:8444" || b.port != 9001 || len(b.tlsConfig.Certificates) != 1 || b.tlsConfig.RootCAs == nil || b.tlsConfig.ServerName != "router.example" { + t.Errorf("bridge = %+v", b) + } + // Dialer is wired to the real network. + if _, err := b.dial(context.Background(), "tcp", "127.0.0.1:1"); err == nil { + t.Error("dial to a closed port succeeded") + } +} + +func TestSubstrateReplaceDirErrors(t *testing.T) { + var good bytes.Buffer + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "f"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := substratebridge.Pack(src, &good); err != nil { + t.Fatal(err) + } + // Parent of the workspace is not writable for the staging dir. + if err := substrateReplaceDir(bytes.NewReader(good.Bytes()), "/nonexistent-parent/ws"); err == nil { + t.Error("staging in a missing parent succeeded") + } + // Workspace itself missing: rename of the old tree fails and staging is removed. + parent := t.TempDir() + if err := substrateReplaceDir(bytes.NewReader(good.Bytes()), filepath.Join(parent, "missing")); err == nil { + t.Error("replacing a missing workspace succeeded") + } + if entries, _ := os.ReadDir(parent); len(entries) != 0 { + t.Errorf("staging left behind: %v", entries) + } +} + +// Preflight. + +type fakeSubstrateFactory struct { + fake *fakeSubstrate + err error +} + +func (f fakeSubstrateFactory) new(SubstrateOptions) (substrateClient, error) { + if f.err != nil { + return nil, f.err + } + return f.fake, nil +} + +func worker(name, state string, labels map[string]string, classes ...string) *substrateapi.Worker { + w := &substrateapi.Worker{Metadata: &substrateapi.ResourceMetadata{Name: name}, Labels: labels, Status: &substrateapi.WorkerStatus{}} + switch state { + case "active": + w.Status.State = substrateapi.WorkerState_WORKER_STATE_ACTIVE + default: + w.Status.State = substrateapi.WorkerState_WORKER_STATE_UNSPECIFIED + } + for _, c := range classes { + w.Status.SandboxClasses = append(w.Status.SandboxClasses, &substrateapi.WorkerSandboxClass{SandboxClass: c}) + } + return w +} + +func TestSubstratePreflighterWarnings(t *testing.T) { + ctx := context.Background() + opts := SubstrateOptions{Endpoint: "api:443", Token: "t"} + + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{err: errors.New("no bundle")}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "unreachable") || !strings.Contains(got[0], "no bundle") { + t.Errorf("constructor failure: %q", got) + } + + fake := &fakeSubstrate{listErr: errors.New("connection refused")} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("list failure: %q", got) + } + if !fake.closed { + t.Error("client not closed after probe") + } + + fake = &fakeSubstrate{} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "no active workers") { + t.Errorf("no workers: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", map[string]string{"class": "microvm"}, "microvm")}} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + sel := opts + sel.WorkerSelector = map[string]string{"class": "gvisor"} + if got := p.Check(ctx, sel); len(got) != 1 || !strings.Contains(got[0], "worker_selector") { + t.Errorf("selector mismatch: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil)}} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "none reports a sandbox class") { + t.Errorf("no class: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{ + worker("down", "unspecified", nil, "gvisor"), + worker("up", "active", map[string]string{"class": "gvisor"}, "GVISOR", ""), + }} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 0 { + t.Errorf("healthy control plane warned: %q", got) + } +} + +func TestSubstratePreflighterCachesPerControlPlane(t *testing.T) { + ctx := context.Background() + fake := &fakeSubstrate{listErr: errors.New("down")} + now := time.Unix(1_700_000_000, 0) + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new, now: func() time.Time { return now }} + a := SubstrateOptions{Endpoint: "a:443", Token: "t"} + b := SubstrateOptions{Endpoint: "b:443", Token: "t"} + p.Check(ctx, a) + p.Check(ctx, a) + p.Check(ctx, b) + if n := len(fake.calls); n != 2 { + t.Errorf("ListWorkers calls = %d, want 2 (one per control plane, second a cache hit)", n) + } + now = now.Add(substratePreflightTTL + time.Second) + p.Check(ctx, a) + if n := len(fake.calls); n != 3 { + t.Errorf("ListWorkers calls after TTL = %d, want 3", n) + } + // Returned slices are copies: mutating one must not poison the cache. + got := p.Check(ctx, a) + got[0] = "mutated" + if again := p.Check(ctx, a); again[0] == "mutated" { + t.Error("cache returned its own backing slice") + } +} + +func TestSubstrateActiveWorkersAndLabels(t *testing.T) { + n, classes := substrateActiveWorkers([]*substrateapi.Worker{ + worker("a", "active", map[string]string{"k": "v"}, "gvisor"), + worker("b", "active", map[string]string{"k": "v"}, "microvm", "gvisor"), + worker("c", "active", map[string]string{"k": "other"}, "firecracker"), + nil, + }, map[string]string{"k": "v"}) + if n != 2 || strings.Join(classes, ",") != "gvisor,microvm" { + t.Errorf("active=%d classes=%v", n, classes) + } + if !substrateLabelsMatch(nil, nil) || substrateLabelsMatch(nil, map[string]string{"a": "b"}) { + t.Error("label matching") + } +} + +func TestDefaultSubstratePreflighterUsesRealClient(t *testing.T) { + p := &SubstratePreflighter{} + got := p.Check(context.Background(), SubstrateOptions{Endpoint: "127.0.0.1:1", CAFile: "/nonexistent/ca.crt"}) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("warnings = %q", got) + } +} diff --git a/src/crux/sandbox/substrate_preflight.go b/src/crux/sandbox/substrate_preflight.go new file mode 100644 index 0000000000..94964cc5e9 --- /dev/null +++ b/src/crux/sandbox/substrate_preflight.go @@ -0,0 +1,125 @@ +package sandbox + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Substrate cluster-prerequisite preflight (ADR-0026 phase 2; the sibling +// of the OpenShell preflight in openshell_preflight.go). Hive cannot see the +// node kernel or the WorkerPool CRDs from its pod without cluster RBAC, but +// the Control API answers whether it is up and which workers — and so which +// sandbox classes — exist; that is what decides whether a kick can land. + +const substratePreflightTTL = 5 * time.Minute + +type substratePreflightEntry struct { + at time.Time + warnings []string +} + +// SubstratePreflighter checks a control plane and remembers the answer for a +// TTL, so the dashboard polling the same warnings does not hit it per call. +type SubstratePreflighter struct { + mu sync.Mutex + cache map[string]substratePreflightEntry + + newClient func(SubstrateOptions) (substrateClient, error) + now func() time.Time +} + +// DefaultSubstratePreflighter backs the warnings AgentSandboxGateWarnings emits. +var DefaultSubstratePreflighter = &SubstratePreflighter{} + +// Check returns the warnings for one control plane. Never an error: an +// unreachable control plane is itself a warning. +func (p *SubstratePreflighter) Check(ctx context.Context, opts SubstrateOptions) []string { + now := time.Now + if p.now != nil { + now = p.now + } + key := opts.endpoint() + "\x00" + opts.atespace() + "\x00" + opts.Token + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.cache[key]; ok && now().Sub(e.at) < substratePreflightTTL { + return append([]string(nil), e.warnings...) + } + warnings := p.probe(ctx, opts) + if p.cache == nil { + p.cache = map[string]substratePreflightEntry{} + } + p.cache[key] = substratePreflightEntry{at: now(), warnings: warnings} + return append([]string(nil), warnings...) +} + +func (p *SubstratePreflighter) probe(ctx context.Context, opts SubstrateOptions) []string { + ep := opts.endpoint() + ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + + newClient := newSubstrateGRPCClient + if p.newClient != nil { + newClient = p.newClient + } + client, err := newClient(opts) + if err != nil { + return []string{fmt.Sprintf("substrate control api %s is unreachable (%v); every kick on the substrate runtime will fail", ep, err)} + } + defer func() { _ = client.Close() }() + + workers, err := client.Control().ListWorkers(ctx) + if err != nil { + return []string{fmt.Sprintf("substrate control api %s is unreachable (%v); every kick on the substrate runtime will fail", ep, err)} + } + active, classes := substrateActiveWorkers(workers, opts.WorkerSelector) + if active == 0 { + if len(opts.WorkerSelector) > 0 { + return []string{fmt.Sprintf("substrate control api %s has no active worker matching worker_selector %v; actors will never be scheduled — check the WorkerPool labels and that its pods are running", ep, opts.WorkerSelector)} + } + return []string{fmt.Sprintf("substrate control api %s reports no active workers; actors will never be scheduled — install a WorkerPool (see deploy/substrate.md) and check its pods are running", ep)} + } + var out []string + if len(classes) == 0 { + out = append(out, fmt.Sprintf("substrate control api %s has %d active worker(s) but none reports a sandbox class; the WorkerPool's SandboxConfig may not have resolved (kubectl get workerpool -A)", ep, active)) + } + return out +} + +// substrateActiveWorkers counts the ACTIVE workers matching the selector and +// returns the sorted set of sandbox classes they advertise. +func substrateActiveWorkers(workers []*substrateapi.Worker, selector map[string]string) (int, []string) { + active := 0 + classSet := map[string]bool{} + for _, w := range workers { + if w.GetStatus().GetState() != substrateapi.WorkerState_WORKER_STATE_ACTIVE || !substrateLabelsMatch(w.GetLabels(), selector) { + continue + } + active++ + for _, c := range w.GetStatus().GetSandboxClasses() { + if c.GetSandboxClass() != "" { + classSet[strings.ToLower(c.GetSandboxClass())] = true + } + } + } + classes := make([]string, 0, len(classSet)) + for c := range classSet { + classes = append(classes, c) + } + sort.Strings(classes) + return active, classes +} + +func substrateLabelsMatch(labels, selector map[string]string) bool { + for k, v := range selector { + if labels[k] != v { + return false + } + } + return true +} diff --git a/src/crux/sandbox/substrate_spec.go b/src/crux/sandbox/substrate_spec.go new file mode 100644 index 0000000000..d4df1e8a16 --- /dev/null +++ b/src/crux/sandbox/substrate_spec.go @@ -0,0 +1,234 @@ +package sandbox + +import ( + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// SubstrateDefaultBridgeCommand is the actor entrypoint when the operator +// names none: the bridge subcommand of the crux binary every hive sandbox +// image ships (ADR-0026 phase 2). +var SubstrateDefaultBridgeCommand = []string{"/usr/local/bin/crux", "substrate-bridge"} + +// SubstrateAlwaysAllowedHosts are the egress hosts every hive actor gets: +// the same GitHub hosts hive's MITM proxy (ADR-0002) brokers for a Podman +// sandbox and the OpenShell policy allows. Substrate's atenet-egress has no +// per-method read-only preset, so api.github.com is allowed as TLS +// passthrough like github.com; the actor still holds no token, and hive does +// every write itself after the kick. +var SubstrateAlwaysAllowedHosts = []string{"github.com", "api.github.com"} + +// substrateImageDigest accepts only digest-pinned references. +var substrateImageDigest = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +// actorTemplate maps the hive LaunchSpec onto a per-kick ActorTemplate: one +// container running the bridge, the workspace as a durableDir, golden +// snapshots off (a template used once has nothing to warm), and the +// operator's resource limits and pool selector. +func (l *SubstrateLauncher) actorTemplate(ref *substrateapi.ObjectRef, image string) *substrateapi.ActorTemplate { + command := l.Options.Command + if len(command) == 0 { + command = SubstrateDefaultBridgeCommand + } + cpu, mem := l.Options.CPUMillicores, l.Options.MemoryBytes + if cpu == 0 { + cpu = SubstrateDefaultCPUMillicores + } + if mem == 0 { + mem = SubstrateDefaultMemoryBytes + } + env := []*substrateapi.EnvVar{{Name: "HIVE_SUBSTRATE_BRIDGE_PORT", Value: strconv.Itoa(l.Options.bridgePort())}} + if p := strings.TrimSpace(l.Options.PolicyProfile); p != "" { + // Substrate has no annotations; the profile travels as an env var the + // bridge and in-actor tooling can read, and as a selector label an + // operator can pin a pool on. + env = append(env, &substrateapi.EnvVar{Name: "HIVE_POLICY_PROFILE", Value: p}) + } + return &substrateapi.ActorTemplate{ + Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + WorkerSelector: substrateSelector(l.Options.WorkerSelector), + Containers: []*substrateapi.Container{{ + Name: substrateContainer, + Image: image, + Command: command, + Env: env, + VolumeMounts: []*substrateapi.VolumeMount{{Name: substrateVolume, MountPath: SubstrateWorkspaceMount}}, + }}, + Volumes: []*substrateapi.Volume{{Name: substrateVolume, DurableDir: &substrateapi.DurableDirVolumeSource{}}}, + SnapshotConfig: &substrateapi.SnapshotConfig{GoldenSnapshotConfig: &substrateapi.GoldenSnapshotConfig{Mode: substrateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}}, + Resources: &substrateapi.ResourceRequirements{Limits: &substrateapi.Resources{CpuMillicores: cpu, MemoryBytes: mem}}, + } +} + +func substrateSelector(labels map[string]string) *substrateapi.Selector { + if len(labels) == 0 { + return nil + } + out := make(map[string]string, len(labels)) + for k, v := range labels { + out[k] = v + } + return &substrateapi.Selector{MatchLabels: out} +} + +// substrateEnv is the environment the kick runs with: the allowlisted, +// credential-filtered subset of hive's own, plus the spec's explicit values — +// the same rule the Podman, Job and OpenShell launchers apply. +func substrateEnv(spec LaunchSpec) map[string]string { + env := map[string]string{} + for _, entry := range SanitizedEnv(os.Environ(), spec.EnvAllowlist, spec.Env) { + if k, v, ok := strings.Cut(entry, "="); ok { + env[k] = v + } + } + return env +} + +// SubstrateEgressPolicy derives the actor's egress policy from hive's +// always-allowed GitHub hosts plus the operator's extra hosts, all as TLS +// passthrough on their port: the actor talks TLS end to end to the same hosts +// hive's MITM proxy would broker for a Podman sandbox, and atenet-egress +// default-denies everything else (including all UDP). Hosts are sorted so the +// policy hive sends is stable across kicks. +func SubstrateEgressPolicy(allowHosts []string) (*substrateapi.EgressPolicy, error) { + byPort := map[int32][]string{} + seen := map[string]bool{} + add := func(host string, port int32) { + key := host + ":" + strconv.Itoa(int(port)) + if seen[key] { + return + } + seen[key] = true + byPort[port] = append(byPort[port], host) + } + for _, h := range SubstrateAlwaysAllowedHosts { + add(h, substrateHTTPSPort) + } + for _, h := range allowHosts { + host, port, err := substrateHostPort(h) + if err != nil { + return nil, err + } + add(host, int32(port)) + } + ports := make([]int32, 0, len(byPort)) + for p := range byPort { + ports = append(ports, p) + } + sort.Slice(ports, func(i, j int) bool { return ports[i] < ports[j] }) + rules := make([]*substrateapi.EgressRule, 0, len(ports)) + for _, p := range ports { + hosts := byPort[p] + sort.Strings(hosts) + rules = append(rules, &substrateapi.EgressRule{TlsPassthrough: &substrateapi.TlsPassthroughRule{ + Hostnames: hosts, + Ports: &substrateapi.Ports{Numbers: []int32{p}}, + }}) + } + return &substrateapi.EgressPolicy{ + Metadata: &substrateapi.ResourceMetadata{Name: substrateEgressPolicy}, + Rules: rules, + }, nil +} + +// substrateAddress turns endpoint into a gRPC host:port target and whether +// TLS is implied. Accepts `host:port`, `http://host:port`, `https://host[:port]` +// and the grpc(s) spellings; a bare host:port is TLS, which is what the +// Control API serves. +func substrateAddress(raw string) (addr string, tls bool, err error) { + raw = strings.TrimSpace(raw) + if !strings.Contains(raw, "://") { + if _, _, err := substrateHostPort(raw); err != nil || !strings.Contains(raw, ":") { + return "", false, fmt.Errorf("substrate: endpoint %q must be host:port or an http(s) URL", raw) + } + return raw, true, nil + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return "", false, fmt.Errorf("substrate: endpoint %q is not a valid URL", raw) + } + switch u.Scheme { + case "https", "grpcs": + tls = true + case "http", "grpc": + default: + return "", false, fmt.Errorf("substrate: endpoint %q has unsupported scheme %q", raw, u.Scheme) + } + host := u.Host + if u.Port() == "" { + if tls { + host += ":443" + } else { + host += ":80" + } + } + return host, tls, nil +} + +// substrateRouterAddress validates router_url and returns its host:port; the +// router serves TLS only, so https is the one accepted scheme. +func substrateRouterAddress(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || u.Scheme != "https" { + return "", fmt.Errorf("substrate: router_url %q must be an https URL (atenet-router serves TLS only)", raw) + } + host := u.Host + if u.Port() == "" { + host += ":443" + } + return host, nil +} + +// substrateHostName is a literal DNS name or a leftmost-wildcard DNS name, +// the two shapes Substrate's egress hostnames accept. +var substrateHostName = regexp.MustCompile(`^(\*\.)?[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$`) + +// substrateHostPort parses an allow_hosts entry: `host` means 443. +func substrateHostPort(raw string) (string, uint32, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", 0, errors.New("substrate: allow_hosts entry is empty") + } + if strings.Contains(raw, "://") || strings.ContainsAny(raw, "/ ") { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q must be host or host:port, not a URL", raw) + } + host, portStr, found := strings.Cut(raw, ":") + if !substrateHostName.MatchString(host) { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q must be a DNS name with at most a leading *. wildcard — no userinfo, query or fragment", raw) + } + host = strings.ToLower(host) + if !found { + return host, substrateHTTPSPort, nil + } + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil || port == 0 { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q has an invalid port", raw) + } + return host, uint32(port), nil +} + +var substrateNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// substrateActorName builds a unique, DNS-label-safe actor (and template) +// name from the launch name and a timestamp: hive-kick--. +func substrateActorName(launchName string, now time.Time) string { + base := strings.Trim(substrateNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + if base == "" { + base = "agent" + } + suffix := fmt.Sprintf("-%d", now.Unix()) + room := substrateNameMax - len(substrateNamePrefix) - len(suffix) + if len(base) > room { + base = strings.TrimRight(base[:room], "-") + } + return substrateNamePrefix + base + suffix +} diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go new file mode 100644 index 0000000000..f306ab9d39 --- /dev/null +++ b/src/crux/sandbox/substrate_test.go @@ -0,0 +1,652 @@ +package sandbox + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// fakeSubstrate is an in-memory stand-in for the Control API slice and the +// in-actor bridge the launcher drives. It records the calls so tests can +// assert on the template, actor, policy and cleanup that reached the control +// plane, and lets each step fail or report a state on demand. +type fakeSubstrate struct { + mu sync.Mutex + + templates []*substrateapi.ActorTemplate + actors []*substrateapi.Actor + policies []*substrateapi.EgressPolicy + calls []string + templateErr error + actorErr error + policyErr error + resumeErr error + suspendErr error + deleteErr error + getErr error + states []substrateapi.ActorState + crashMsg string + getCancels context.CancelFunc + waitReadyErr error + pushErr error + runErr error + runResp substratebridge.RunResponse + runCancels context.CancelFunc + pullErr error + runReq substratebridge.RunRequest + pushedWorkspace string + pulledWorkspace string + workers []*substrateapi.Worker + listErr error + closed bool + cleanupCtxDead bool +} + +func (f *fakeSubstrate) note(c string, ctx context.Context) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, c) + if (c == "DeleteActor" || c == "DeleteActorTemplate" || c == "SuspendActor") && ctx.Err() != nil { + f.cleanupCtxDead = true + } +} + +func (f *fakeSubstrate) Control() substrateControl { return f } +func (f *fakeSubstrate) Bridge() substrateBridge { return f } +func (f *fakeSubstrate) Close() error { f.closed = true; return nil } + +func (f *fakeSubstrate) CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) { + f.note("CreateActorTemplate", ctx) + if f.templateErr != nil { + return nil, f.templateErr + } + f.templates = append(f.templates, tpl) + return tpl, nil +} + +func (f *fakeSubstrate) DeleteActorTemplate(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("DeleteActorTemplate", ctx) + return nil +} + +func (f *fakeSubstrate) CreateActor(ctx context.Context, a *substrateapi.Actor) (*substrateapi.Actor, error) { + f.note("CreateActor", ctx) + if f.actorErr != nil { + return nil, f.actorErr + } + f.actors = append(f.actors, a) + return a, nil +} + +func (f *fakeSubstrate) GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) { + f.note("GetActor", ctx) + if f.getErr != nil { + return nil, f.getErr + } + state := substrateapi.ActorState_ACTOR_STATE_RUNNING + if len(f.states) > 0 { + state, f.states = f.states[0], f.states[1:] + } + if f.getCancels != nil { + f.getCancels() + } + return &substrateapi.Actor{ + Metadata: &substrateapi.ResourceMetadata{Name: ref.GetName()}, + Status: &substrateapi.ActorStatus{State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}}, + }, nil +} + +func (f *fakeSubstrate) ResumeActor(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("ResumeActor", ctx) + return f.resumeErr +} + +func (f *fakeSubstrate) SuspendActor(ctx context.Context, _ *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error { + f.note("SuspendActor", ctx) + f.calls = append(f.calls, "mode="+mode.String()) + return f.suspendErr +} + +func (f *fakeSubstrate) DeleteActor(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("DeleteActor", ctx) + return f.deleteErr +} + +func (f *fakeSubstrate) CreateActorEgressPolicy(ctx context.Context, _ *substrateapi.ObjectRef, p *substrateapi.EgressPolicy) error { + f.note("CreateActorEgressPolicy", ctx) + if f.policyErr != nil { + return f.policyErr + } + f.policies = append(f.policies, p) + return nil +} + +func (f *fakeSubstrate) ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) { + f.note("ListWorkers", ctx) + return f.workers, f.listErr +} + +func (f *fakeSubstrate) WaitReady(ctx context.Context, _ *substrateapi.ObjectRef, _ time.Duration) error { + f.note("WaitReady", ctx) + return f.waitReadyErr +} + +func (f *fakeSubstrate) Push(ctx context.Context, _ *substrateapi.ObjectRef, ws string) error { + f.note("Push", ctx) + f.pushedWorkspace = ws + return f.pushErr +} + +func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { + f.note("Run", ctx) + f.runReq = req + if f.runCancels != nil { + f.runCancels() + } + return f.runResp, f.runErr +} + +func (f *fakeSubstrate) Pull(ctx context.Context, _ *substrateapi.ObjectRef, ws string) error { + f.note("Pull", ctx) + f.pulledWorkspace = ws + return f.pullErr +} + +func (f *fakeSubstrate) saw(call string) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, c := range f.calls { + if c == call { + return true + } + } + return false +} + +const substrateTestImage = "registry.example/hive/sandbox@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func newTestSubstrate(fake *fakeSubstrate) *SubstrateLauncher { + l := &SubstrateLauncher{Options: SubstrateOptions{ + Endpoint: "api.ate-system.svc:443", + Atespace: "hive", + Token: "tok", + Image: substrateTestImage, + AllowHosts: []string{"litellm.example.com", "*.pypi.org:8443"}, + }} + l.newClient = func(SubstrateOptions) (substrateClient, error) { return fake, nil } + l.now = func() time.Time { return time.Unix(1_700_000_000, 0) } + return l +} + +func substrateTestSpec() LaunchSpec { + return LaunchSpec{ + Image: "registry.example/hive/sandbox:dev", + Name: "deps-sandbox", + Workspace: "/data/agents/sandbox/deps", + Command: []string{"crux", "kick", "--agent", "deps"}, + Env: map[string]string{"HIVE_AGENT": "deps"}, + } +} + +func TestSubstrateRunHappyPath(t *testing.T) { + fake := &fakeSubstrate{runResp: substratebridge.RunResponse{Stdout: "done\n", Stderr: "warn\n", ExitCode: 0}} + l := newTestSubstrate(fake) + l.Options.PolicyProfile = "strict" + l.Options.WorkerSelector = map[string]string{"class": "gvisor"} + res, err := l.Run(context.Background(), substrateTestSpec()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Stdout != "done\n" || res.Stderr != "warn\n" || res.ExitCode != 0 { + t.Errorf("result = %+v", res) + } + want := []string{"CreateActorTemplate", "CreateActor", "CreateActorEgressPolicy", "ResumeActor", "GetActor", "WaitReady", "Push", "Run", "Pull", "DeleteActor", "DeleteActorTemplate"} + if got := strings.Join(fake.calls, " "); got != strings.Join(want, " ") { + t.Errorf("calls = %s\nwant = %s", got, strings.Join(want, " ")) + } + if !fake.closed { + t.Error("client not closed") + } + if len(fake.templates) != 1 { + t.Fatalf("templates = %d", len(fake.templates)) + } + tpl := fake.templates[0] + if tpl.GetMetadata().GetName() != "hive-kick-deps-1700000000" || tpl.GetMetadata().GetAtespace() != "hive" { + t.Errorf("template metadata = %v", tpl.GetMetadata()) + } + c := tpl.GetContainers()[0] + if c.GetImage() != substrateTestImage || strings.Join(c.GetCommand(), " ") != strings.Join(SubstrateDefaultBridgeCommand, " ") { + t.Errorf("container = %v", c) + } + var envNames []string + for _, e := range c.GetEnv() { + envNames = append(envNames, e.GetName()+"="+e.GetValue()) + } + if joined := strings.Join(envNames, ","); !strings.Contains(joined, "HIVE_SUBSTRATE_BRIDGE_PORT=8477") || !strings.Contains(joined, "HIVE_POLICY_PROFILE=strict") { + t.Errorf("env = %s", joined) + } + if tpl.GetWorkerSelector().GetMatchLabels()["class"] != "gvisor" || fake.actors[0].GetWorkerSelector().GetMatchLabels()["class"] != "gvisor" { + t.Error("worker selector not propagated to template and actor") + } + if tpl.GetVolumes()[0].GetDurableDir() == nil || c.GetVolumeMounts()[0].GetMountPath() != SubstrateWorkspaceMount { + t.Errorf("workspace volume = %v / %v", tpl.GetVolumes(), c.GetVolumeMounts()) + } + if tpl.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() != substrateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { + t.Error("golden snapshot not disabled") + } + if lim := tpl.GetResources().GetLimits(); lim.GetCpuMillicores() != SubstrateDefaultCPUMillicores || lim.GetMemoryBytes() != SubstrateDefaultMemoryBytes { + t.Errorf("limits = %v", lim) + } + if fake.actors[0].GetActorTemplate().GetName() != tpl.GetMetadata().GetName() { + t.Error("actor does not reference the per-kick template") + } + if fake.pushedWorkspace != "/data/agents/sandbox/deps" || fake.pulledWorkspace != "/data/agents/sandbox/deps" { + t.Errorf("workspace push/pull = %q / %q", fake.pushedWorkspace, fake.pulledWorkspace) + } + if fake.runReq.WorkDir != SubstrateWorkspaceMount || strings.Join(fake.runReq.Command, " ") != "crux kick --agent deps" || fake.runReq.Env["HIVE_AGENT"] != "deps" { + t.Errorf("run request = %+v", fake.runReq) + } + // Egress: GitHub hosts plus the operator's, grouped by port, sorted. + pol := fake.policies[0] + if pol.GetMetadata().GetName() != "default" || len(pol.GetRules()) != 2 { + t.Fatalf("policy = %v", pol) + } + r443 := pol.GetRules()[0].GetTlsPassthrough() + if strings.Join(r443.GetHostnames(), ",") != "api.github.com,github.com,litellm.example.com" || r443.GetPorts().GetNumbers()[0] != 443 { + t.Errorf("443 rule = %v", r443) + } + r8443 := pol.GetRules()[1].GetTlsPassthrough() + if strings.Join(r8443.GetHostnames(), ",") != "*.pypi.org" || r8443.GetPorts().GetNumbers()[0] != 8443 { + t.Errorf("8443 rule = %v", r8443) + } + if fake.cleanupCtxDead { + t.Error("cleanup ran on a dead context although the caller's was live") + } +} + +func TestSubstrateRunHonoursSpecOverrides(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.Image = "" + l.Options.Command = []string{"/opt/bridge"} + l.Options.CPUMillicores = 500 + l.Options.MemoryBytes = 1 << 30 + l.Options.BridgePort = 9000 + l.Options.Timeout = time.Minute + spec := substrateTestSpec() + spec.Image = substrateTestImage + spec.WorkDir = "/workspace/sub" + spec.NetworkNone = true + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + c := fake.templates[0].GetContainers()[0] + if c.GetImage() != substrateTestImage || c.GetCommand()[0] != "/opt/bridge" || c.GetEnv()[0].GetValue() != "9000" { + t.Errorf("container = %v", c) + } + if lim := fake.templates[0].GetResources().GetLimits(); lim.GetCpuMillicores() != 500 || lim.GetMemoryBytes() != 1<<30 { + t.Errorf("limits = %v", lim) + } + if fake.templates[0].GetWorkerSelector() != nil { + t.Error("empty worker selector must be nil") + } + if len(fake.policies[0].GetRules()) != 0 { + t.Errorf("network_none must send an empty policy, got %v", fake.policies[0].GetRules()) + } + if fake.runReq.WorkDir != "/workspace/sub" { + t.Errorf("workdir = %q", fake.runReq.WorkDir) + } +} + +func TestSubstrateRunRefusesClineProvider(t *testing.T) { + spec, _ := clineLaunchFixture(t) + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + spec.Workspace = "/data/agents/sandbox/deps" + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "Podman-only") { + t.Fatalf("err = %v", err) + } + if len(fake.calls) != 0 { + t.Error("control plane touched despite the Cline provider refusal") + } +} + +func TestSubstrateRunRejectsBadOptionsAndSpec(t *testing.T) { + cases := []struct { + name string + mut func(l *SubstrateLauncher, spec *LaunchSpec) + want string + }{ + {"bad endpoint", func(l *SubstrateLauncher, _ *LaunchSpec) { l.Options.Endpoint = "ftp://x" }, "unsupported scheme"}, + {"tag image in options", func(l *SubstrateLauncher, _ *LaunchSpec) { l.Options.Image = "img:latest" }, "pinned by digest"}, + {"no image anywhere", func(l *SubstrateLauncher, s *LaunchSpec) { l.Options.Image = ""; s.Image = "" }, "image is required"}, + {"tag image in spec", func(l *SubstrateLauncher, s *LaunchSpec) { l.Options.Image = ""; s.Image = "img:latest" }, "pinned by digest"}, + {"no workspace", func(_ *SubstrateLauncher, s *LaunchSpec) { s.Workspace = " " }, "workspace is required"}, + {"no command", func(_ *SubstrateLauncher, s *LaunchSpec) { s.Command = nil }, "command is required"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + spec := substrateTestSpec() + tc.mut(l, &spec) + _, err := l.Run(context.Background(), spec) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + if len(fake.calls) != 0 { + t.Errorf("control plane touched: %v", fake.calls) + } + }) + } +} + +func TestSubstrateRunClientConstructorError(t *testing.T) { + l := newTestSubstrate(&fakeSubstrate{}) + l.newClient = func(SubstrateOptions) (substrateClient, error) { return nil, errors.New("no router bundle") } + if _, err := l.Run(context.Background(), substrateTestSpec()); err == nil || !strings.Contains(err.Error(), "no router bundle") { + t.Fatalf("err = %v", err) + } +} + +func TestSubstrateRunControlPlaneFailuresCleanUp(t *testing.T) { + cases := []struct { + name string + mut func(f *fakeSubstrate) + want string + cleanup []string + }{ + {"template", func(f *fakeSubstrate) { f.templateErr = errors.New("boom") }, "creating actor template", nil}, + {"actor", func(f *fakeSubstrate) { f.actorErr = errors.New("boom") }, "creating actor hive-kick", []string{"DeleteActorTemplate"}}, + {"policy", func(f *fakeSubstrate) { f.policyErr = errors.New("boom") }, "setting egress policy", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"resume", func(f *fakeSubstrate) { f.resumeErr = errors.New("boom") }, "resuming actor", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"get actor", func(f *fakeSubstrate) { f.getErr = errors.New("boom") }, "never became running", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"crashed", func(f *fakeSubstrate) { + f.states = []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_CRASHED} + f.crashMsg = "OOM" + }, "actor crashed: OOM", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"deleting", func(f *fakeSubstrate) { + f.states = []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_DELETING} + }, "entered ACTOR_STATE_DELETING", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"bridge never ready", func(f *fakeSubstrate) { f.waitReadyErr = errors.New("boom") }, "never answered", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"push", func(f *fakeSubstrate) { f.pushErr = errors.New("boom") }, "pushing workspace", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"run transport", func(f *fakeSubstrate) { f.runErr = errors.New("boom") }, "running kick", []string{"Pull", "DeleteActor", "DeleteActorTemplate"}}, + {"run reported", func(f *fakeSubstrate) { + f.runResp = substratebridge.RunResponse{ExitCode: -1, Error: "exec: not found"} + }, "exec: not found", []string{"Pull", "DeleteActor", "DeleteActorTemplate"}}, + {"pull", func(f *fakeSubstrate) { f.pullErr = errors.New("boom") }, "pulling workspace", []string{"DeleteActor", "DeleteActorTemplate"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeSubstrate{} + tc.mut(fake) + l := newTestSubstrate(fake) + _, err := l.Run(context.Background(), substrateTestSpec()) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + for _, c := range tc.cleanup { + if !fake.saw(c) { + t.Errorf("cleanup %s not called: %v", c, fake.calls) + } + } + if tc.name == "template" && fake.saw("DeleteActorTemplate") { + t.Error("template deleted although it was never created") + } + if !fake.closed { + t.Error("client not closed") + } + }) + } +} + +func TestSubstrateRunReportsNonZeroExit(t *testing.T) { + fake := &fakeSubstrate{runResp: substratebridge.RunResponse{Stdout: "x", ExitCode: 3}} + res, err := newTestSubstrate(fake).Run(context.Background(), substrateTestSpec()) + if err != nil { + t.Fatalf("Run: %v (a non-zero exit is a result, not an error)", err) + } + if res.ExitCode != 3 || res.Stdout != "x" { + t.Errorf("result = %+v", res) + } +} + +func TestSubstrateRunSnapshotOnIdle(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if !fake.saw("SuspendActor") || !fake.saw("mode=DURABILITY_MODE_ASYNCHRONOUS") || fake.saw("DeleteActor") { + t.Errorf("snapshot_on_idle must suspend asynchronously instead of deleting: %v", fake.calls) + } + if !fake.saw("DeleteActorTemplate") { + t.Error("per-kick template must still be removed") + } + + // A failed suspend falls back to delete so no actor is left running. + fake = &fakeSubstrate{suspendErr: errors.New("no snapshot store")} + l = newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if !fake.saw("SuspendActor") || !fake.saw("DeleteActor") { + t.Errorf("suspend failure must fall back to delete: %v", fake.calls) + } +} + +func TestSubstrateRunContextCancelledMidKick(t *testing.T) { + // The kick's context dies while the command runs: Pull is skipped, the + // actor is deleted on a fresh context, and the caller gets ctx.Err(). + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{runCancels: cancel} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true // must not snapshot on a dead context + _, err := l.Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if fake.saw("Pull") || fake.saw("SuspendActor") { + t.Errorf("pull or suspend ran after cancel: %v", fake.calls) + } + if !fake.saw("DeleteActor") || !fake.saw("DeleteActorTemplate") { + t.Errorf("cleanup skipped: %v", fake.calls) + } + if fake.cleanupCtxDead { + t.Error("cleanup used the dead caller context instead of a fresh one") + } +} + +func TestSubstrateRunContextCancelledWhileWaitingForRunning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{getCancels: cancel, states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_RESUMING}} + _, err := newTestSubstrate(fake).Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if !fake.saw("DeleteActor") || fake.saw("WaitReady") { + t.Errorf("calls = %v", fake.calls) + } +} + +func TestSubstrateRunContextCancelledWhileWaitingForBridge(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{waitReadyErr: context.Canceled} + l := newTestSubstrate(fake) + // Cancel just before WaitReady by hooking GetActor (the step before). + fake.getCancels = cancel + _, err := l.Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if fake.saw("Push") { + t.Errorf("push ran after cancel: %v", fake.calls) + } +} + +func TestSubstrateWaitRunningPollsUntilRunning(t *testing.T) { + fake := &fakeSubstrate{states: []substrateapi.ActorState{ + substrateapi.ActorState_ACTOR_STATE_SUSPENDED, + substrateapi.ActorState_ACTOR_STATE_RESUMING, + substrateapi.ActorState_ACTOR_STATE_RUNNING, + }} + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err != nil { + t.Fatalf("err = %v", err) + } + if n := len(fake.calls); n != 3 { + t.Errorf("GetActor calls = %d, want 3", n) + } + fake = &fakeSubstrate{states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_SUSPENDING}} + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { + t.Errorf("err = %v", err) + } +} + +func TestSubstrateOptionsValidate(t *testing.T) { + good := SubstrateOptions{Image: substrateTestImage, AllowHosts: []string{"a.example.com", "*.b.example.com:8443"}} + if err := good.Validate(); err != nil { + t.Fatalf("Validate(good) = %v", err) + } + if err := (SubstrateOptions{}).Validate(); err != nil { + t.Fatalf("Validate(defaults) = %v", err) + } + cases := map[string]SubstrateOptions{ + "endpoint no port": {Endpoint: "api.ate-system.svc"}, + "endpoint bad url": {Endpoint: "https://"}, + "endpoint scheme": {Endpoint: "ftp://x:1"}, + "router http": {RouterURL: "http://router:8444"}, + "router garbage": {RouterURL: "::not a url"}, + "image tag": {Image: "img:latest"}, + "negative cpu": {CPUMillicores: -1}, + "negative mem": {MemoryBytes: -1}, + "bridge port range": {BridgePort: 70000}, + "allow host url": {AllowHosts: []string{"https://x.example.com"}}, + "allow host empty": {AllowHosts: []string{" "}}, + "allow host userinfo": {AllowHosts: []string{"u@x.example.com"}}, + "allow host bad port": {AllowHosts: []string{"x.example.com:0"}}, + "allow host port alpha": {AllowHosts: []string{"x.example.com:abc"}}, + "allow host wildcard": {AllowHosts: []string{"*x.example.com"}}, + } + for name, o := range cases { + if err := o.Validate(); err == nil { + t.Errorf("%s: Validate accepted %+v", name, o) + } + } +} + +func TestSubstrateAddressForms(t *testing.T) { + cases := []struct { + in string + addr string + tls bool + }{ + {"api.ate-system.svc:443", "api.ate-system.svc:443", true}, + {"https://api.ate-system.svc", "api.ate-system.svc:443", true}, + {"grpcs://api:9443", "api:9443", true}, + {"http://localhost:8080", "localhost:8080", false}, + {"http://localhost", "localhost:80", false}, + {"grpc://api:50051", "api:50051", false}, + } + for _, tc := range cases { + addr, tls, err := substrateAddress(tc.in) + if err != nil || addr != tc.addr || tls != tc.tls { + t.Errorf("substrateAddress(%q) = %q %v %v", tc.in, addr, tls, err) + } + } + if got, err := substrateRouterAddress("https://atenet-router.ate-system.svc"); err != nil || got != "atenet-router.ate-system.svc:443" { + t.Errorf("router default port: %q %v", got, err) + } + if got, err := substrateRouterAddress("https://r:8444"); err != nil || got != "r:8444" { + t.Errorf("router explicit port: %q %v", got, err) + } +} + +func TestSubstrateHostPort(t *testing.T) { + host, port, err := substrateHostPort("API.Example.COM") + if err != nil || host != "api.example.com" || port != 443 { + t.Errorf("got %q %d %v", host, port, err) + } + host, port, err = substrateHostPort("*.pypi.org:8443") + if err != nil || host != "*.pypi.org" || port != 8443 { + t.Errorf("got %q %d %v", host, port, err) + } + for _, bad := range []string{"", "a b", "x/y", "x.example.com:99999", "-bad.example.com", "x.example.com:"} { + if _, _, err := substrateHostPort(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} + +func TestSubstrateEgressPolicyDedupesAndErrors(t *testing.T) { + pol, err := SubstrateEgressPolicy([]string{"github.com", "GitHub.com:443", "x.example.com:8443"}) + if err != nil { + t.Fatal(err) + } + if len(pol.GetRules()) != 2 || len(pol.GetRules()[0].GetTlsPassthrough().GetHostnames()) != 2 { + t.Errorf("policy = %v", pol) + } + if _, err := SubstrateEgressPolicy([]string{"bad host"}); err == nil { + t.Error("bad host accepted") + } +} + +func TestSubstrateActorName(t *testing.T) { + at := time.Unix(1_700_000_000, 0) + if got := substrateActorName("Deps_Review-sandbox", at); got != "hive-kick-deps-review-1700000000" { + t.Errorf("got %q", got) + } + if got := substrateActorName("---", at); got != "hive-kick-agent-1700000000" { + t.Errorf("empty base: %q", got) + } + long := substrateActorName(strings.Repeat("a", 80)+"-b", at) + if len(long) > substrateNameMax || !strings.HasPrefix(long, "hive-kick-aaaa") || !strings.HasSuffix(long, "-1700000000") { + t.Errorf("long name = %q (%d)", long, len(long)) + } + trimmed := substrateActorName(strings.Repeat("a", 41)+"-"+strings.Repeat("b", 20), at) + if strings.Contains(trimmed, "--") { + t.Errorf("truncation left a dangling dash: %q", trimmed) + } +} + +func TestSubstrateOptionDefaults(t *testing.T) { + o := SubstrateOptions{} + if o.endpoint() != SubstrateDefaultEndpoint || o.atespace() != SubstrateDefaultAtespace || o.routerURL() != SubstrateDefaultRouterURL || o.bridgePort() != substratebridge.DefaultPort || o.tokenFile() != SubstrateDefaultTokenFile { + t.Errorf("defaults = %q %q %q %d %q", o.endpoint(), o.atespace(), o.routerURL(), o.bridgePort(), o.tokenFile()) + } + o = SubstrateOptions{Endpoint: " e:1 ", Atespace: " a ", RouterURL: " https://r ", BridgePort: 1, TokenFile: " /t "} + if o.endpoint() != "e:1" || o.atespace() != "a" || o.routerURL() != "https://r" || o.bridgePort() != 1 || o.tokenFile() != "/t" { + t.Errorf("trimmed = %q %q %q %d %q", o.endpoint(), o.atespace(), o.routerURL(), o.bridgePort(), o.tokenFile()) + } +} + +func TestSubstrateEnvFiltersCredentials(t *testing.T) { + t.Setenv("GITHUB_TOKEN", "ghp_secret") + t.Setenv("HIVE_SUBSTRATE_TEST_KEEP", "yes") + spec := substrateTestSpec() + spec.EnvAllowlist = []string{"HIVE_SUBSTRATE_TEST_KEEP"} + env := substrateEnv(spec) + if env["HIVE_SUBSTRATE_TEST_KEEP"] != "yes" || env["HIVE_AGENT"] != "deps" { + t.Errorf("allowlisted values missing: %v", env) + } + if _, ok := env["GITHUB_TOKEN"]; ok { + t.Error("GITHUB_TOKEN leaked into the actor env") + } +} + +func TestSubstrateLauncherDefaultClientIsGRPC(t *testing.T) { + // Without a test hook the launcher builds the real client; with no + // router bundle on disk that fails before any network I/O. + l := &SubstrateLauncher{Options: SubstrateOptions{InsecureTLS: true, RouterClientBundle: "/nonexistent/bundle.pem"}} + if _, err := l.client(); err == nil || !strings.Contains(err.Error(), "router_client_bundle") { + t.Fatalf("client() = %v", err) + } +} diff --git a/src/crux/sandbox/substrateapi/actor.pb.go b/src/crux/sandbox/substrateapi/actor.pb.go new file mode 100644 index 0000000000..d70548216f --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor.pb.go @@ -0,0 +1,586 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: actor.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ActorState int32 + +const ( + ActorState_ACTOR_STATE_UNSPECIFIED ActorState = 0 + ActorState_ACTOR_STATE_RESUMING ActorState = 1 + ActorState_ACTOR_STATE_RUNNING ActorState = 2 + ActorState_ACTOR_STATE_SUSPENDING ActorState = 3 + ActorState_ACTOR_STATE_SUSPENDED ActorState = 4 + ActorState_ACTOR_STATE_CRASHED ActorState = 5 + ActorState_ACTOR_STATE_DELETING ActorState = 6 + ActorState_ACTOR_STATE_REVERTING ActorState = 7 +) + +// Enum value maps for ActorState. +var ( + ActorState_name = map[int32]string{ + 0: "ACTOR_STATE_UNSPECIFIED", + 1: "ACTOR_STATE_RESUMING", + 2: "ACTOR_STATE_RUNNING", + 3: "ACTOR_STATE_SUSPENDING", + 4: "ACTOR_STATE_SUSPENDED", + 5: "ACTOR_STATE_CRASHED", + 6: "ACTOR_STATE_DELETING", + 7: "ACTOR_STATE_REVERTING", + } + ActorState_value = map[string]int32{ + "ACTOR_STATE_UNSPECIFIED": 0, + "ACTOR_STATE_RESUMING": 1, + "ACTOR_STATE_RUNNING": 2, + "ACTOR_STATE_SUSPENDING": 3, + "ACTOR_STATE_SUSPENDED": 4, + "ACTOR_STATE_CRASHED": 5, + "ACTOR_STATE_DELETING": 6, + "ACTOR_STATE_REVERTING": 7, + } +) + +func (x ActorState) Enum() *ActorState { + p := new(ActorState) + *p = x + return p +} + +func (x ActorState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ActorState) Descriptor() protoreflect.EnumDescriptor { + return file_actor_proto_enumTypes[0].Descriptor() +} + +func (ActorState) Type() protoreflect.EnumType { + return &file_actor_proto_enumTypes[0] +} + +func (x ActorState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ActorState.Descriptor instead. +func (ActorState) EnumDescriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{0} +} + +type Actor struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + ActorTemplate *ObjectRef `protobuf:"bytes,2,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + WorkerSelector *Selector `protobuf:"bytes,3,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` + SourceTag *ObjectRef `protobuf:"bytes,4,opt,name=source_tag,json=sourceTag,proto3" json:"source_tag,omitempty"` + Status *ActorStatus `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Actor) Reset() { + *x = Actor{} + mi := &file_actor_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Actor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Actor) ProtoMessage() {} + +func (x *Actor) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Actor.ProtoReflect.Descriptor instead. +func (*Actor) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{0} +} + +func (x *Actor) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Actor) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +func (x *Actor) GetWorkerSelector() *Selector { + if x != nil { + return x.WorkerSelector + } + return nil +} + +func (x *Actor) GetSourceTag() *ObjectRef { + if x != nil { + return x.SourceTag + } + return nil +} + +func (x *Actor) GetStatus() *ActorStatus { + if x != nil { + return x.Status + } + return nil +} + +type ActorStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State ActorState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.ActorState" json:"state,omitempty"` + WorkerAssignment *WorkerAssignment `protobuf:"bytes,2,opt,name=worker_assignment,json=workerAssignment,proto3" json:"worker_assignment,omitempty"` + Crash *ActorCrash `protobuf:"bytes,4,opt,name=crash,proto3" json:"crash,omitempty"` + AssignedNode string `protobuf:"bytes,5,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorStatus) Reset() { + *x = ActorStatus{} + mi := &file_actor_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorStatus) ProtoMessage() {} + +func (x *ActorStatus) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorStatus.ProtoReflect.Descriptor instead. +func (*ActorStatus) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{1} +} + +func (x *ActorStatus) GetState() ActorState { + if x != nil { + return x.State + } + return ActorState_ACTOR_STATE_UNSPECIFIED +} + +func (x *ActorStatus) GetWorkerAssignment() *WorkerAssignment { + if x != nil { + return x.WorkerAssignment + } + return nil +} + +func (x *ActorStatus) GetCrash() *ActorCrash { + if x != nil { + return x.Crash + } + return nil +} + +func (x *ActorStatus) GetAssignedNode() string { + if x != nil { + return x.AssignedNode + } + return "" +} + +type ActorCrash struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + CrashTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=crash_time,json=crashTime,proto3" json:"crash_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorCrash) Reset() { + *x = ActorCrash{} + mi := &file_actor_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorCrash) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorCrash) ProtoMessage() {} + +func (x *ActorCrash) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorCrash.ProtoReflect.Descriptor instead. +func (*ActorCrash) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{2} +} + +func (x *ActorCrash) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *ActorCrash) GetCrashTime() *timestamppb.Timestamp { + if x != nil { + return x.CrashTime + } + return nil +} + +type WorkerAssignment struct { + state protoimpl.MessageState `protogen:"open.v1"` + Worker *ObjectRef `protobuf:"bytes,6,opt,name=worker,proto3" json:"worker,omitempty"` + WorkerNamespace string `protobuf:"bytes,1,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` + WorkerPool string `protobuf:"bytes,2,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` + WorkerPod string `protobuf:"bytes,3,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` + WorkerPodUid string `protobuf:"bytes,4,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` + WorkerPodIps []string `protobuf:"bytes,5,rep,name=worker_pod_ips,json=workerPodIps,proto3" json:"worker_pod_ips,omitempty"` + NodeName string `protobuf:"bytes,7,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + WorkerEpoch int64 `protobuf:"varint,8,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` + SandboxRuntime *ActorSandboxRuntime `protobuf:"bytes,9,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,10,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerAssignment) Reset() { + *x = WorkerAssignment{} + mi := &file_actor_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerAssignment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerAssignment) ProtoMessage() {} + +func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerAssignment.ProtoReflect.Descriptor instead. +func (*WorkerAssignment) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{3} +} + +func (x *WorkerAssignment) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +func (x *WorkerAssignment) GetWorkerNamespace() string { + if x != nil { + return x.WorkerNamespace + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPool() string { + if x != nil { + return x.WorkerPool + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPod() string { + if x != nil { + return x.WorkerPod + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPodUid() string { + if x != nil { + return x.WorkerPodUid + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPodIps() []string { + if x != nil { + return x.WorkerPodIps + } + return nil +} + +func (x *WorkerAssignment) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +func (x *WorkerAssignment) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 +} + +func (x *WorkerAssignment) GetSandboxRuntime() *ActorSandboxRuntime { + if x != nil { + return x.SandboxRuntime + } + return nil +} + +func (x *WorkerAssignment) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + +type ActorSandboxRuntime struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorSandboxRuntime) Reset() { + *x = ActorSandboxRuntime{} + mi := &file_actor_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorSandboxRuntime) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorSandboxRuntime) ProtoMessage() {} + +func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. +func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{4} +} + +func (x *ActorSandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +var File_actor_proto protoreflect.FileDescriptor + +const file_actor_proto_rawDesc = "" + + "\n" + + "\vactor.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xd2\x02\n" + + "\x05Actor\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12E\n" + + "\x0eactor_template\x18\x02 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\x12F\n" + + "\x0fworker_selector\x18\x03 \x01(\v2\x1d.substrate.ateapi.v1.SelectorR\x0eworkerSelector\x12=\n" + + "\n" + + "source_tag\x18\x04 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\tsourceTag\x128\n" + + "\x06status\x18\x05 \x01(\v2 .substrate.ateapi.v1.ActorStatusR\x06status\"\xf4\x01\n" + + "\vActorStatus\x125\n" + + "\x05state\x18\x01 \x01(\x0e2\x1f.substrate.ateapi.v1.ActorStateR\x05state\x12R\n" + + "\x11worker_assignment\x18\x02 \x01(\v2%.substrate.ateapi.v1.WorkerAssignmentR\x10workerAssignment\x125\n" + + "\x05crash\x18\x04 \x01(\v2\x1f.substrate.ateapi.v1.ActorCrashR\x05crash\x12#\n" + + "\rassigned_node\x18\x05 \x01(\tR\fassignedNode\"a\n" + + "\n" + + "ActorCrash\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\x129\n" + + "\n" + + "crash_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\tcrashTime\"\xbe\x03\n" + + "\x10WorkerAssignment\x126\n" + + "\x06worker\x18\x06 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x06worker\x12)\n" + + "\x10worker_namespace\x18\x01 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + + "\vworker_pool\x18\x02 \x01(\tR\n" + + "workerPool\x12\x1d\n" + + "\n" + + "worker_pod\x18\x03 \x01(\tR\tworkerPod\x12$\n" + + "\x0eworker_pod_uid\x18\x04 \x01(\tR\fworkerPodUid\x12$\n" + + "\x0eworker_pod_ips\x18\x05 \x03(\tR\fworkerPodIps\x12\x1b\n" + + "\tnode_name\x18\a \x01(\tR\bnodeName\x12!\n" + + "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\x12Q\n" + + "\x0fsandbox_runtime\x18\t \x01(\v2(.substrate.ateapi.v1.ActorSandboxRuntimeR\x0esandboxRuntime\x12(\n" + + "\x10worker_spiffe_id\x18\n" + + " \x01(\tR\x0eworkerSpiffeId\":\n" + + "\x13ActorSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass*\xe1\x01\n" + + "\n" + + "ActorState\x12\x1b\n" + + "\x17ACTOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14ACTOR_STATE_RESUMING\x10\x01\x12\x17\n" + + "\x13ACTOR_STATE_RUNNING\x10\x02\x12\x1a\n" + + "\x16ACTOR_STATE_SUSPENDING\x10\x03\x12\x19\n" + + "\x15ACTOR_STATE_SUSPENDED\x10\x04\x12\x17\n" + + "\x13ACTOR_STATE_CRASHED\x10\x05\x12\x18\n" + + "\x14ACTOR_STATE_DELETING\x10\x06\x12\x19\n" + + "\x15ACTOR_STATE_REVERTING\x10\aBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_actor_proto_rawDescOnce sync.Once + file_actor_proto_rawDescData []byte +) + +func file_actor_proto_rawDescGZIP() []byte { + file_actor_proto_rawDescOnce.Do(func() { + file_actor_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_actor_proto_rawDesc), len(file_actor_proto_rawDesc))) + }) + return file_actor_proto_rawDescData +} + +var file_actor_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_actor_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_actor_proto_goTypes = []any{ + (ActorState)(0), // 0: substrate.ateapi.v1.ActorState + (*Actor)(nil), // 1: substrate.ateapi.v1.Actor + (*ActorStatus)(nil), // 2: substrate.ateapi.v1.ActorStatus + (*ActorCrash)(nil), // 3: substrate.ateapi.v1.ActorCrash + (*WorkerAssignment)(nil), // 4: substrate.ateapi.v1.WorkerAssignment + (*ActorSandboxRuntime)(nil), // 5: substrate.ateapi.v1.ActorSandboxRuntime + (*ResourceMetadata)(nil), // 6: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 7: substrate.ateapi.v1.ObjectRef + (*Selector)(nil), // 8: substrate.ateapi.v1.Selector + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp +} +var file_actor_proto_depIdxs = []int32{ + 6, // 0: substrate.ateapi.v1.Actor.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 7, // 1: substrate.ateapi.v1.Actor.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 8, // 2: substrate.ateapi.v1.Actor.worker_selector:type_name -> substrate.ateapi.v1.Selector + 7, // 3: substrate.ateapi.v1.Actor.source_tag:type_name -> substrate.ateapi.v1.ObjectRef + 2, // 4: substrate.ateapi.v1.Actor.status:type_name -> substrate.ateapi.v1.ActorStatus + 0, // 5: substrate.ateapi.v1.ActorStatus.state:type_name -> substrate.ateapi.v1.ActorState + 4, // 6: substrate.ateapi.v1.ActorStatus.worker_assignment:type_name -> substrate.ateapi.v1.WorkerAssignment + 3, // 7: substrate.ateapi.v1.ActorStatus.crash:type_name -> substrate.ateapi.v1.ActorCrash + 9, // 8: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 7, // 9: substrate.ateapi.v1.WorkerAssignment.worker:type_name -> substrate.ateapi.v1.ObjectRef + 5, // 10: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_actor_proto_init() } +func file_actor_proto_init() { + if File_actor_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_actor_proto_rawDesc), len(file_actor_proto_rawDesc)), + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_actor_proto_goTypes, + DependencyIndexes: file_actor_proto_depIdxs, + EnumInfos: file_actor_proto_enumTypes, + MessageInfos: file_actor_proto_msgTypes, + }.Build() + File_actor_proto = out.File + file_actor_proto_goTypes = nil + file_actor_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/actor.proto b/src/crux/sandbox/substrateapi/actor.proto new file mode 100644 index 0000000000..d5df52af03 --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor.proto @@ -0,0 +1,79 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Actor { + ResourceMetadata metadata = 1; + ObjectRef actor_template = 2; + Selector worker_selector = 3; + ObjectRef source_tag = 4; + ActorStatus status = 5; +} + +enum ActorState { + ACTOR_STATE_UNSPECIFIED = 0; + ACTOR_STATE_RESUMING = 1; + ACTOR_STATE_RUNNING = 2; + ACTOR_STATE_SUSPENDING = 3; + ACTOR_STATE_SUSPENDED = 4; + ACTOR_STATE_CRASHED = 5; + ACTOR_STATE_DELETING = 6; + ACTOR_STATE_REVERTING = 7; +} + +message ActorStatus { + ActorState state = 1; + WorkerAssignment worker_assignment = 2; + ActorCrash crash = 4; + string assigned_node = 5; +} + +message ActorCrash { + string message = 1; + google.protobuf.Timestamp crash_time = 2; +} + +message WorkerAssignment { + ObjectRef worker = 6; + string worker_namespace = 1; + string worker_pool = 2; + string worker_pod = 3; + string worker_pod_uid = 4; + repeated string worker_pod_ips = 5; + string node_name = 7; + int64 worker_epoch = 8; + ActorSandboxRuntime sandbox_runtime = 9; + string worker_spiffe_id = 10; +} + +message ActorSandboxRuntime { + string sandbox_class = 1; +} diff --git a/src/crux/sandbox/substrateapi/actor_rpc.pb.go b/src/crux/sandbox/substrateapi/actor_rpc.pb.go new file mode 100644 index 0000000000..17d81ba51d --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor_rpc.pb.go @@ -0,0 +1,572 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: actor_rpc.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DurabilityMode int32 + +const ( + DurabilityMode_DURABILITY_MODE_UNSPECIFIED DurabilityMode = 0 + DurabilityMode_DURABILITY_MODE_SYNCHRONOUS DurabilityMode = 1 + DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS DurabilityMode = 2 +) + +// Enum value maps for DurabilityMode. +var ( + DurabilityMode_name = map[int32]string{ + 0: "DURABILITY_MODE_UNSPECIFIED", + 1: "DURABILITY_MODE_SYNCHRONOUS", + 2: "DURABILITY_MODE_ASYNCHRONOUS", + } + DurabilityMode_value = map[string]int32{ + "DURABILITY_MODE_UNSPECIFIED": 0, + "DURABILITY_MODE_SYNCHRONOUS": 1, + "DURABILITY_MODE_ASYNCHRONOUS": 2, + } +) + +func (x DurabilityMode) Enum() *DurabilityMode { + p := new(DurabilityMode) + *p = x + return p +} + +func (x DurabilityMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DurabilityMode) Descriptor() protoreflect.EnumDescriptor { + return file_actor_rpc_proto_enumTypes[0].Descriptor() +} + +func (DurabilityMode) Type() protoreflect.EnumType { + return &file_actor_rpc_proto_enumTypes[0] +} + +func (x DurabilityMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DurabilityMode.Descriptor instead. +func (DurabilityMode) EnumDescriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{0} +} + +type GetActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetActorRequest) Reset() { + *x = GetActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetActorRequest) ProtoMessage() {} + +func (x *GetActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetActorRequest.ProtoReflect.Descriptor instead. +func (*GetActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{0} +} + +func (x *GetActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +type CreateActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorRequest) Reset() { + *x = CreateActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorRequest) ProtoMessage() {} + +func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorRequest.ProtoReflect.Descriptor instead. +func (*CreateActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{1} +} + +func (x *CreateActorRequest) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +type SuspendActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Options *SuspendOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorRequest) Reset() { + *x = SuspendActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorRequest) ProtoMessage() {} + +func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorRequest.ProtoReflect.Descriptor instead. +func (*SuspendActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{2} +} + +func (x *SuspendActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *SuspendActorRequest) GetOptions() *SuspendOptions { + if x != nil { + return x.Options + } + return nil +} + +type SuspendActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorResponse) Reset() { + *x = SuspendActorResponse{} + mi := &file_actor_rpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorResponse) ProtoMessage() {} + +func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorResponse.ProtoReflect.Descriptor instead. +func (*SuspendActorResponse) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{3} +} + +func (x *SuspendActorResponse) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +type SuspendOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode DurabilityMode `protobuf:"varint,1,opt,name=mode,proto3,enum=substrate.ateapi.v1.DurabilityMode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendOptions) Reset() { + *x = SuspendOptions{} + mi := &file_actor_rpc_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendOptions) ProtoMessage() {} + +func (x *SuspendOptions) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendOptions.ProtoReflect.Descriptor instead. +func (*SuspendOptions) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{4} +} + +func (x *SuspendOptions) GetMode() DurabilityMode { + if x != nil { + return x.Mode + } + return DurabilityMode_DURABILITY_MODE_UNSPECIFIED +} + +type ResumeActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResumeActorRequest) Reset() { + *x = ResumeActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResumeActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeActorRequest) ProtoMessage() {} + +func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. +func (*ResumeActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{5} +} + +func (x *ResumeActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +type ResumeActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Resumed bool `protobuf:"varint,2,opt,name=resumed,proto3" json:"resumed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResumeActorResponse) Reset() { + *x = ResumeActorResponse{} + mi := &file_actor_rpc_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResumeActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeActorResponse) ProtoMessage() {} + +func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. +func (*ResumeActorResponse) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{6} +} + +func (x *ResumeActorResponse) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +func (x *ResumeActorResponse) GetResumed() bool { + if x != nil { + return x.Resumed + } + return false +} + +type DeleteActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteActorRequest) Reset() { + *x = DeleteActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteActorRequest) ProtoMessage() {} + +func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. +func (*DeleteActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{7} +} + +func (x *DeleteActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *DeleteActorRequest) GetOptions() *DeleteOptions { + if x != nil { + return x.Options + } + return nil +} + +var File_actor_rpc_proto protoreflect.FileDescriptor + +const file_actor_rpc_proto_rawDesc = "" + + "\n" + + "\x0factor_rpc.proto\x12\x13substrate.ateapi.v1\x1a\vactor.proto\x1a\fcommon.proto\"G\n" + + "\x0fGetActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\"F\n" + + "\x12CreateActorRequest\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\"\x8a\x01\n" + + "\x13SuspendActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12=\n" + + "\aoptions\x18\x02 \x01(\v2#.substrate.ateapi.v1.SuspendOptionsR\aoptions\"H\n" + + "\x14SuspendActorResponse\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\"I\n" + + "\x0eSuspendOptions\x127\n" + + "\x04mode\x18\x01 \x01(\x0e2#.substrate.ateapi.v1.DurabilityModeR\x04mode\"J\n" + + "\x12ResumeActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\"a\n" + + "\x13ResumeActorResponse\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\x12\x18\n" + + "\aresumed\x18\x02 \x01(\bR\aresumed\"\x88\x01\n" + + "\x12DeleteActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12<\n" + + "\aoptions\x18\x02 \x01(\v2\".substrate.ateapi.v1.DeleteOptionsR\aoptions*t\n" + + "\x0eDurabilityMode\x12\x1f\n" + + "\x1bDURABILITY_MODE_UNSPECIFIED\x10\x00\x12\x1f\n" + + "\x1bDURABILITY_MODE_SYNCHRONOUS\x10\x01\x12 \n" + + "\x1cDURABILITY_MODE_ASYNCHRONOUS\x10\x02BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_actor_rpc_proto_rawDescOnce sync.Once + file_actor_rpc_proto_rawDescData []byte +) + +func file_actor_rpc_proto_rawDescGZIP() []byte { + file_actor_rpc_proto_rawDescOnce.Do(func() { + file_actor_rpc_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_actor_rpc_proto_rawDesc), len(file_actor_rpc_proto_rawDesc))) + }) + return file_actor_rpc_proto_rawDescData +} + +var file_actor_rpc_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_actor_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_actor_rpc_proto_goTypes = []any{ + (DurabilityMode)(0), // 0: substrate.ateapi.v1.DurabilityMode + (*GetActorRequest)(nil), // 1: substrate.ateapi.v1.GetActorRequest + (*CreateActorRequest)(nil), // 2: substrate.ateapi.v1.CreateActorRequest + (*SuspendActorRequest)(nil), // 3: substrate.ateapi.v1.SuspendActorRequest + (*SuspendActorResponse)(nil), // 4: substrate.ateapi.v1.SuspendActorResponse + (*SuspendOptions)(nil), // 5: substrate.ateapi.v1.SuspendOptions + (*ResumeActorRequest)(nil), // 6: substrate.ateapi.v1.ResumeActorRequest + (*ResumeActorResponse)(nil), // 7: substrate.ateapi.v1.ResumeActorResponse + (*DeleteActorRequest)(nil), // 8: substrate.ateapi.v1.DeleteActorRequest + (*ObjectRef)(nil), // 9: substrate.ateapi.v1.ObjectRef + (*Actor)(nil), // 10: substrate.ateapi.v1.Actor + (*DeleteOptions)(nil), // 11: substrate.ateapi.v1.DeleteOptions +} +var file_actor_rpc_proto_depIdxs = []int32{ + 9, // 0: substrate.ateapi.v1.GetActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 10, // 1: substrate.ateapi.v1.CreateActorRequest.actor:type_name -> substrate.ateapi.v1.Actor + 9, // 2: substrate.ateapi.v1.SuspendActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 5, // 3: substrate.ateapi.v1.SuspendActorRequest.options:type_name -> substrate.ateapi.v1.SuspendOptions + 10, // 4: substrate.ateapi.v1.SuspendActorResponse.actor:type_name -> substrate.ateapi.v1.Actor + 0, // 5: substrate.ateapi.v1.SuspendOptions.mode:type_name -> substrate.ateapi.v1.DurabilityMode + 9, // 6: substrate.ateapi.v1.ResumeActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 10, // 7: substrate.ateapi.v1.ResumeActorResponse.actor:type_name -> substrate.ateapi.v1.Actor + 9, // 8: substrate.ateapi.v1.DeleteActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 11, // 9: substrate.ateapi.v1.DeleteActorRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_actor_rpc_proto_init() } +func file_actor_rpc_proto_init() { + if File_actor_rpc_proto != nil { + return + } + file_actor_proto_init() + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_actor_rpc_proto_rawDesc), len(file_actor_rpc_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_actor_rpc_proto_goTypes, + DependencyIndexes: file_actor_rpc_proto_depIdxs, + EnumInfos: file_actor_rpc_proto_enumTypes, + MessageInfos: file_actor_rpc_proto_msgTypes, + }.Build() + File_actor_rpc_proto = out.File + file_actor_rpc_proto_goTypes = nil + file_actor_rpc_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/actor_rpc.proto b/src/crux/sandbox/substrateapi/actor_rpc.proto new file mode 100644 index 0000000000..1e81978bc1 --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor_rpc.proto @@ -0,0 +1,72 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "actor.proto"; +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message GetActorRequest { + ObjectRef actor = 1; +} + +message CreateActorRequest { + Actor actor = 1; +} + +message SuspendActorRequest { + ObjectRef actor = 1; + SuspendOptions options = 2; +} + +message SuspendActorResponse { + Actor actor = 1; +} + +message SuspendOptions { + DurabilityMode mode = 1; +} + +enum DurabilityMode { + DURABILITY_MODE_UNSPECIFIED = 0; + DURABILITY_MODE_SYNCHRONOUS = 1; + DURABILITY_MODE_ASYNCHRONOUS = 2; +} + +message ResumeActorRequest { + ObjectRef actor = 1; +} + +message ResumeActorResponse { + Actor actor = 1; + bool resumed = 2; +} + +message DeleteActorRequest { + ObjectRef actor = 1; + DeleteOptions options = 2; +} diff --git a/src/crux/sandbox/substrateapi/buf.gen.yaml b/src/crux/sandbox/substrateapi/buf.gen.yaml new file mode 100644 index 0000000000..c5ca49b889 --- /dev/null +++ b/src/crux/sandbox/substrateapi/buf.gen.yaml @@ -0,0 +1,8 @@ +version: v2 +plugins: + - local: protoc-gen-go + out: . + opt: paths=source_relative + - local: protoc-gen-go-grpc + out: . + opt: paths=source_relative diff --git a/src/crux/sandbox/substrateapi/buf.yaml b/src/crux/sandbox/substrateapi/buf.yaml new file mode 100644 index 0000000000..b8699818a0 --- /dev/null +++ b/src/crux/sandbox/substrateapi/buf.yaml @@ -0,0 +1,3 @@ +version: v2 +modules: + - path: . diff --git a/src/crux/sandbox/substrateapi/common.pb.go b/src/crux/sandbox/substrateapi/common.pb.go new file mode 100644 index 0000000000..02d6dd3697 --- /dev/null +++ b/src/crux/sandbox/substrateapi/common.pb.go @@ -0,0 +1,428 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: common.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + MatchLabels map[string]string `protobuf:"bytes,1,rep,name=match_labels,json=matchLabels,proto3" json:"match_labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{0} +} + +func (x *Selector) GetMatchLabels() map[string]string { + if x != nil { + return x.MatchLabels + } + return nil +} + +type ResourceMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Uid string `protobuf:"bytes,3,opt,name=uid,proto3" json:"uid,omitempty"` + Version int64 `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"` + CreateTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"` + UpdateTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=update_time,json=updateTime,proto3" json:"update_time,omitempty"` + DeleteTime *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=delete_time,json=deleteTime,proto3" json:"delete_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceMetadata) Reset() { + *x = ResourceMetadata{} + mi := &file_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceMetadata) ProtoMessage() {} + +func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceMetadata.ProtoReflect.Descriptor instead. +func (*ResourceMetadata) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{1} +} + +func (x *ResourceMetadata) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *ResourceMetadata) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ResourceMetadata) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +func (x *ResourceMetadata) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ResourceMetadata) GetCreateTime() *timestamppb.Timestamp { + if x != nil { + return x.CreateTime + } + return nil +} + +func (x *ResourceMetadata) GetUpdateTime() *timestamppb.Timestamp { + if x != nil { + return x.UpdateTime + } + return nil +} + +func (x *ResourceMetadata) GetDeleteTime() *timestamppb.Timestamp { + if x != nil { + return x.DeleteTime + } + return nil +} + +type ObjectRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectRef) Reset() { + *x = ObjectRef{} + mi := &file_common_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectRef) ProtoMessage() {} + +func (x *ObjectRef) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectRef.ProtoReflect.Descriptor instead. +func (*ObjectRef) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{2} +} + +func (x *ObjectRef) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *ObjectRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type DeleteOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteOptions) Reset() { + *x = DeleteOptions{} + mi := &file_common_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteOptions) ProtoMessage() {} + +func (x *DeleteOptions) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. +func (*DeleteOptions) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{3} +} + +func (x *DeleteOptions) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *DeleteOptions) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +type ListOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListOptions) Reset() { + *x = ListOptions{} + mi := &file_common_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListOptions) ProtoMessage() {} + +func (x *ListOptions) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListOptions.ProtoReflect.Descriptor instead. +func (*ListOptions) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{4} +} + +func (x *ListOptions) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListOptions) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +var File_common_proto protoreflect.FileDescriptor + +const file_common_proto_rawDesc = "" + + "\n" + + "\fcommon.proto\x12\x13substrate.ateapi.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\x9d\x01\n" + + "\bSelector\x12Q\n" + + "\fmatch_labels\x18\x01 \x03(\v2..substrate.ateapi.v1.Selector.MatchLabelsEntryR\vmatchLabels\x1a>\n" + + "\x10MatchLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xa5\x02\n" + + "\x10ResourceMetadata\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x10\n" + + "\x03uid\x18\x03 \x01(\tR\x03uid\x12\x18\n" + + "\aversion\x18\x04 \x01(\x03R\aversion\x12;\n" + + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "createTime\x12;\n" + + "\vupdate_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "updateTime\x12;\n" + + "\vdelete_time\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "deleteTime\";\n" + + "\tObjectRef\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\";\n" + + "\rDeleteOptions\x12\x18\n" + + "\aversion\x18\x01 \x01(\x03R\aversion\x12\x10\n" + + "\x03uid\x18\x02 \x01(\tR\x03uid\"I\n" + + "\vListOptions\x12\x1b\n" + + "\tpage_size\x18\x01 \x01(\x05R\bpageSize\x12\x1d\n" + + "\n" + + "page_token\x18\x02 \x01(\tR\tpageTokenBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_common_proto_rawDescOnce sync.Once + file_common_proto_rawDescData []byte +) + +func file_common_proto_rawDescGZIP() []byte { + file_common_proto_rawDescOnce.Do(func() { + file_common_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc))) + }) + return file_common_proto_rawDescData +} + +var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_common_proto_goTypes = []any{ + (*Selector)(nil), // 0: substrate.ateapi.v1.Selector + (*ResourceMetadata)(nil), // 1: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 2: substrate.ateapi.v1.ObjectRef + (*DeleteOptions)(nil), // 3: substrate.ateapi.v1.DeleteOptions + (*ListOptions)(nil), // 4: substrate.ateapi.v1.ListOptions + nil, // 5: substrate.ateapi.v1.Selector.MatchLabelsEntry + (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp +} +var file_common_proto_depIdxs = []int32{ + 5, // 0: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry + 6, // 1: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 6, // 2: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 6, // 3: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_common_proto_init() } +func file_common_proto_init() { + if File_common_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc)), + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_common_proto_goTypes, + DependencyIndexes: file_common_proto_depIdxs, + MessageInfos: file_common_proto_msgTypes, + }.Build() + File_common_proto = out.File + file_common_proto_goTypes = nil + file_common_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/common.proto b/src/crux/sandbox/substrateapi/common.proto new file mode 100644 index 0000000000..9ca13d7696 --- /dev/null +++ b/src/crux/sandbox/substrateapi/common.proto @@ -0,0 +1,59 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Selector { + map match_labels = 1; +} + +message ResourceMetadata { + string atespace = 1; + string name = 2; + string uid = 3; + int64 version = 4; + google.protobuf.Timestamp create_time = 5; + google.protobuf.Timestamp update_time = 6; + google.protobuf.Timestamp delete_time = 7; +} + +message ObjectRef { + string atespace = 1; + string name = 2; +} + +message DeleteOptions { + int64 version = 1; + string uid = 2; +} + +message ListOptions { + int32 page_size = 1; + string page_token = 2; +} diff --git a/src/crux/sandbox/substrateapi/container.pb.go b/src/crux/sandbox/substrateapi/container.pb.go new file mode 100644 index 0000000000..7389ce3037 --- /dev/null +++ b/src/crux/sandbox/substrateapi/container.pb.go @@ -0,0 +1,401 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: container.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Container struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Image string `protobuf:"bytes,2,opt,name=image,proto3" json:"image,omitempty"` + Command []string `protobuf:"bytes,3,rep,name=command,proto3" json:"command,omitempty"` + Args []string `protobuf:"bytes,4,rep,name=args,proto3" json:"args,omitempty"` + Env []*EnvVar `protobuf:"bytes,5,rep,name=env,proto3" json:"env,omitempty"` + VolumeMounts []*VolumeMount `protobuf:"bytes,7,rep,name=volume_mounts,json=volumeMounts,proto3" json:"volume_mounts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Container) Reset() { + *x = Container{} + mi := &file_container_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Container) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Container) ProtoMessage() {} + +func (x *Container) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Container.ProtoReflect.Descriptor instead. +func (*Container) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{0} +} + +func (x *Container) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Container) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + +func (x *Container) GetCommand() []string { + if x != nil { + return x.Command + } + return nil +} + +func (x *Container) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *Container) GetEnv() []*EnvVar { + if x != nil { + return x.Env + } + return nil +} + +func (x *Container) GetVolumeMounts() []*VolumeMount { + if x != nil { + return x.VolumeMounts + } + return nil +} + +type EnvVar struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvVar) Reset() { + *x = EnvVar{} + mi := &file_container_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvVar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvVar) ProtoMessage() {} + +func (x *EnvVar) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. +func (*EnvVar) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{1} +} + +func (x *EnvVar) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *EnvVar) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +type Volume struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + DurableDir *DurableDirVolumeSource `protobuf:"bytes,2,opt,name=durable_dir,json=durableDir,proto3" json:"durable_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Volume) Reset() { + *x = Volume{} + mi := &file_container_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Volume) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Volume) ProtoMessage() {} + +func (x *Volume) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Volume.ProtoReflect.Descriptor instead. +func (*Volume) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{2} +} + +func (x *Volume) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Volume) GetDurableDir() *DurableDirVolumeSource { + if x != nil { + return x.DurableDir + } + return nil +} + +type DurableDirVolumeSource struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DurableDirVolumeSource) Reset() { + *x = DurableDirVolumeSource{} + mi := &file_container_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DurableDirVolumeSource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DurableDirVolumeSource) ProtoMessage() {} + +func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. +func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{3} +} + +type VolumeMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeMount) Reset() { + *x = VolumeMount{} + mi := &file_container_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeMount) ProtoMessage() {} + +func (x *VolumeMount) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. +func (*VolumeMount) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{4} +} + +func (x *VolumeMount) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *VolumeMount) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + +var File_container_proto protoreflect.FileDescriptor + +const file_container_proto_rawDesc = "" + + "\n" + + "\x0fcontainer.proto\x12\x13substrate.ateapi.v1\"\xd9\x01\n" + + "\tContainer\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05image\x18\x02 \x01(\tR\x05image\x12\x18\n" + + "\acommand\x18\x03 \x03(\tR\acommand\x12\x12\n" + + "\x04args\x18\x04 \x03(\tR\x04args\x12-\n" + + "\x03env\x18\x05 \x03(\v2\x1b.substrate.ateapi.v1.EnvVarR\x03env\x12E\n" + + "\rvolume_mounts\x18\a \x03(\v2 .substrate.ateapi.v1.VolumeMountR\fvolumeMounts\"7\n" + + "\x06EnvVar\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" + + "\x05value\x18\x02 \x01(\tB\x03\x80\x01\x01R\x05value\"j\n" + + "\x06Volume\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12L\n" + + "\vdurable_dir\x18\x02 \x01(\v2+.substrate.ateapi.v1.DurableDirVolumeSourceR\n" + + "durableDir\"\x18\n" + + "\x16DurableDirVolumeSource\"@\n" + + "\vVolumeMount\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + + "\n" + + "mount_path\x18\x02 \x01(\tR\tmountPathBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_container_proto_rawDescOnce sync.Once + file_container_proto_rawDescData []byte +) + +func file_container_proto_rawDescGZIP() []byte { + file_container_proto_rawDescOnce.Do(func() { + file_container_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_container_proto_rawDesc), len(file_container_proto_rawDesc))) + }) + return file_container_proto_rawDescData +} + +var file_container_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_container_proto_goTypes = []any{ + (*Container)(nil), // 0: substrate.ateapi.v1.Container + (*EnvVar)(nil), // 1: substrate.ateapi.v1.EnvVar + (*Volume)(nil), // 2: substrate.ateapi.v1.Volume + (*DurableDirVolumeSource)(nil), // 3: substrate.ateapi.v1.DurableDirVolumeSource + (*VolumeMount)(nil), // 4: substrate.ateapi.v1.VolumeMount +} +var file_container_proto_depIdxs = []int32{ + 1, // 0: substrate.ateapi.v1.Container.env:type_name -> substrate.ateapi.v1.EnvVar + 4, // 1: substrate.ateapi.v1.Container.volume_mounts:type_name -> substrate.ateapi.v1.VolumeMount + 3, // 2: substrate.ateapi.v1.Volume.durable_dir:type_name -> substrate.ateapi.v1.DurableDirVolumeSource + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_container_proto_init() } +func file_container_proto_init() { + if File_container_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_container_proto_rawDesc), len(file_container_proto_rawDesc)), + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_container_proto_goTypes, + DependencyIndexes: file_container_proto_depIdxs, + MessageInfos: file_container_proto_msgTypes, + }.Build() + File_container_proto = out.File + file_container_proto_goTypes = nil + file_container_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/container.proto b/src/crux/sandbox/substrateapi/container.proto new file mode 100644 index 0000000000..d4d2dbf37f --- /dev/null +++ b/src/crux/sandbox/substrateapi/container.proto @@ -0,0 +1,54 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Container { + string name = 1; + string image = 2; + repeated string command = 3; + repeated string args = 4; + repeated EnvVar env = 5; + repeated VolumeMount volume_mounts = 7; +} + +message EnvVar { + string name = 1; + string value = 2 [ debug_redact = true ]; +} + +message Volume { + string name = 1; + DurableDirVolumeSource durable_dir = 2; +} + +message DurableDirVolumeSource {} + +message VolumeMount { + string name = 1; + string mount_path = 2; +} diff --git a/src/crux/sandbox/substrateapi/control.pb.go b/src/crux/sandbox/substrateapi/control.pb.go new file mode 100644 index 0000000000..5dab339ffb --- /dev/null +++ b/src/crux/sandbox/substrateapi/control.pb.go @@ -0,0 +1,135 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: control.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_control_proto protoreflect.FileDescriptor + +const file_control_proto_rawDesc = "" + + "\n" + + "\rcontrol.proto\x12\x13substrate.ateapi.v1\x1a\vactor.proto\x1a\x0factor_rpc.proto\x1a\fegress.proto\x1a\x0etemplate.proto\x1a\fworker.proto2\xed\a\n" + + "\aControl\x12N\n" + + "\bGetActor\x12$.substrate.ateapi.v1.GetActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12T\n" + + "\vCreateActor\x12'.substrate.ateapi.v1.CreateActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12e\n" + + "\fSuspendActor\x12(.substrate.ateapi.v1.SuspendActorRequest\x1a).substrate.ateapi.v1.SuspendActorResponse\"\x00\x12b\n" + + "\vResumeActor\x12'.substrate.ateapi.v1.ResumeActorRequest\x1a(.substrate.ateapi.v1.ResumeActorResponse\"\x00\x12T\n" + + "\vDeleteActor\x12'.substrate.ateapi.v1.DeleteActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12s\n" + + "\x17CreateActorEgressPolicy\x123.substrate.ateapi.v1.CreateActorEgressPolicyRequest\x1a!.substrate.ateapi.v1.EgressPolicy\"\x00\x12b\n" + + "\vListWorkers\x12'.substrate.ateapi.v1.ListWorkersRequest\x1a(.substrate.ateapi.v1.ListWorkersResponse\"\x00\x12l\n" + + "\x13CreateActorTemplate\x12/.substrate.ateapi.v1.CreateActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00\x12f\n" + + "\x10GetActorTemplate\x12,.substrate.ateapi.v1.GetActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00\x12l\n" + + "\x13DeleteActorTemplate\x12/.substrate.ateapi.v1.DeleteActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var file_control_proto_goTypes = []any{ + (*GetActorRequest)(nil), // 0: substrate.ateapi.v1.GetActorRequest + (*CreateActorRequest)(nil), // 1: substrate.ateapi.v1.CreateActorRequest + (*SuspendActorRequest)(nil), // 2: substrate.ateapi.v1.SuspendActorRequest + (*ResumeActorRequest)(nil), // 3: substrate.ateapi.v1.ResumeActorRequest + (*DeleteActorRequest)(nil), // 4: substrate.ateapi.v1.DeleteActorRequest + (*CreateActorEgressPolicyRequest)(nil), // 5: substrate.ateapi.v1.CreateActorEgressPolicyRequest + (*ListWorkersRequest)(nil), // 6: substrate.ateapi.v1.ListWorkersRequest + (*CreateActorTemplateRequest)(nil), // 7: substrate.ateapi.v1.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 8: substrate.ateapi.v1.GetActorTemplateRequest + (*DeleteActorTemplateRequest)(nil), // 9: substrate.ateapi.v1.DeleteActorTemplateRequest + (*Actor)(nil), // 10: substrate.ateapi.v1.Actor + (*SuspendActorResponse)(nil), // 11: substrate.ateapi.v1.SuspendActorResponse + (*ResumeActorResponse)(nil), // 12: substrate.ateapi.v1.ResumeActorResponse + (*EgressPolicy)(nil), // 13: substrate.ateapi.v1.EgressPolicy + (*ListWorkersResponse)(nil), // 14: substrate.ateapi.v1.ListWorkersResponse + (*ActorTemplate)(nil), // 15: substrate.ateapi.v1.ActorTemplate +} +var file_control_proto_depIdxs = []int32{ + 0, // 0: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest + 1, // 1: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest + 2, // 2: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest + 3, // 3: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest + 4, // 4: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest + 5, // 5: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest + 6, // 6: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest + 7, // 7: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest + 8, // 8: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest + 9, // 9: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest + 10, // 10: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor + 10, // 11: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor + 11, // 12: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse + 12, // 13: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse + 10, // 14: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor + 13, // 15: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 14, // 16: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse + 15, // 17: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 15, // 18: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 15, // 19: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 10, // [10:20] is the sub-list for method output_type + 0, // [0:10] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_control_proto_init() } +func file_control_proto_init() { + if File_control_proto != nil { + return + } + file_actor_proto_init() + file_actor_rpc_proto_init() + file_egress_proto_init() + file_template_proto_init() + file_worker_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_control_proto_rawDesc), len(file_control_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_control_proto_goTypes, + DependencyIndexes: file_control_proto_depIdxs, + }.Build() + File_control_proto = out.File + file_control_proto_goTypes = nil + file_control_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/control.proto b/src/crux/sandbox/substrateapi/control.proto new file mode 100644 index 0000000000..e95d5a6d98 --- /dev/null +++ b/src/crux/sandbox/substrateapi/control.proto @@ -0,0 +1,48 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "actor.proto"; +import "actor_rpc.proto"; +import "egress.proto"; +import "template.proto"; +import "worker.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +// Control is the Agent Substrate control-plane API served by ate-api-server. +service Control { + rpc GetActor(GetActorRequest) returns (Actor) {} + rpc CreateActor(CreateActorRequest) returns (Actor) {} + rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} + rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {} + rpc DeleteActor(DeleteActorRequest) returns (Actor) {} + rpc CreateActorEgressPolicy(CreateActorEgressPolicyRequest) returns (EgressPolicy) {} + rpc ListWorkers(ListWorkersRequest) returns (ListWorkersResponse) {} + rpc CreateActorTemplate(CreateActorTemplateRequest) returns (ActorTemplate) {} + rpc GetActorTemplate(GetActorTemplateRequest) returns (ActorTemplate) {} + rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) {} +} diff --git a/src/crux/sandbox/substrateapi/control_grpc.pb.go b/src/crux/sandbox/substrateapi/control_grpc.pb.go new file mode 100644 index 0000000000..cc4663c9fb --- /dev/null +++ b/src/crux/sandbox/substrateapi/control_grpc.pb.go @@ -0,0 +1,490 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: control.proto + +package substrateapi + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Control_GetActor_FullMethodName = "/substrate.ateapi.v1.Control/GetActor" + Control_CreateActor_FullMethodName = "/substrate.ateapi.v1.Control/CreateActor" + Control_SuspendActor_FullMethodName = "/substrate.ateapi.v1.Control/SuspendActor" + Control_ResumeActor_FullMethodName = "/substrate.ateapi.v1.Control/ResumeActor" + Control_DeleteActor_FullMethodName = "/substrate.ateapi.v1.Control/DeleteActor" + Control_CreateActorEgressPolicy_FullMethodName = "/substrate.ateapi.v1.Control/CreateActorEgressPolicy" + Control_ListWorkers_FullMethodName = "/substrate.ateapi.v1.Control/ListWorkers" + Control_CreateActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/CreateActorTemplate" + Control_GetActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/GetActorTemplate" + Control_DeleteActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/DeleteActorTemplate" +) + +// ControlClient is the client API for Control service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Control is the Agent Substrate control-plane API served by ate-api-server. +type ControlClient interface { + GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) + CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) + SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) + ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) + DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) + CreateActorEgressPolicy(ctx context.Context, in *CreateActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) + ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) + CreateActorTemplate(ctx context.Context, in *CreateActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) + GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) + DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) +} + +type controlClient struct { + cc grpc.ClientConnInterface +} + +func NewControlClient(cc grpc.ClientConnInterface) ControlClient { + return &controlClient{cc} +} + +func (c *controlClient) GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_GetActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_CreateActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SuspendActorResponse) + err := c.cc.Invoke(ctx, Control_SuspendActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ResumeActorResponse) + err := c.cc.Invoke(ctx, Control_ResumeActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_DeleteActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActorEgressPolicy(ctx context.Context, in *CreateActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EgressPolicy) + err := c.cc.Invoke(ctx, Control_CreateActorEgressPolicy_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListWorkersResponse) + err := c.cc.Invoke(ctx, Control_ListWorkers_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActorTemplate(ctx context.Context, in *CreateActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_CreateActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_GetActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_DeleteActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ControlServer is the server API for Control service. +// All implementations must embed UnimplementedControlServer +// for forward compatibility. +// +// Control is the Agent Substrate control-plane API served by ate-api-server. +type ControlServer interface { + GetActor(context.Context, *GetActorRequest) (*Actor, error) + CreateActor(context.Context, *CreateActorRequest) (*Actor, error) + SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) + ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) + DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) + CreateActorEgressPolicy(context.Context, *CreateActorEgressPolicyRequest) (*EgressPolicy, error) + ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) + CreateActorTemplate(context.Context, *CreateActorTemplateRequest) (*ActorTemplate, error) + GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) + DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) + mustEmbedUnimplementedControlServer() +} + +// UnimplementedControlServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedControlServer struct{} + +func (UnimplementedControlServer) GetActor(context.Context, *GetActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method GetActor not implemented") +} +func (UnimplementedControlServer) CreateActor(context.Context, *CreateActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActor not implemented") +} +func (UnimplementedControlServer) SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SuspendActor not implemented") +} +func (UnimplementedControlServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ResumeActor not implemented") +} +func (UnimplementedControlServer) DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteActor not implemented") +} +func (UnimplementedControlServer) CreateActorEgressPolicy(context.Context, *CreateActorEgressPolicyRequest) (*EgressPolicy, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActorEgressPolicy not implemented") +} +func (UnimplementedControlServer) ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListWorkers not implemented") +} +func (UnimplementedControlServer) CreateActorTemplate(context.Context, *CreateActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActorTemplate not implemented") +} +func (UnimplementedControlServer) GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method GetActorTemplate not implemented") +} +func (UnimplementedControlServer) DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteActorTemplate not implemented") +} +func (UnimplementedControlServer) mustEmbedUnimplementedControlServer() {} +func (UnimplementedControlServer) testEmbeddedByValue() {} + +// UnsafeControlServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ControlServer will +// result in compilation errors. +type UnsafeControlServer interface { + mustEmbedUnimplementedControlServer() +} + +func RegisterControlServer(s grpc.ServiceRegistrar, srv ControlServer) { + // If the following call panics, it indicates UnimplementedControlServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Control_ServiceDesc, srv) +} + +func _Control_GetActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).GetActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_GetActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).GetActor(ctx, req.(*GetActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActor(ctx, req.(*CreateActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_SuspendActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SuspendActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).SuspendActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_SuspendActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).SuspendActor(ctx, req.(*SuspendActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_ResumeActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ResumeActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ResumeActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ResumeActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ResumeActor(ctx, req.(*ResumeActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_DeleteActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).DeleteActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_DeleteActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).DeleteActor(ctx, req.(*DeleteActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActorEgressPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorEgressPolicyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActorEgressPolicy(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActorEgressPolicy_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActorEgressPolicy(ctx, req.(*CreateActorEgressPolicyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_ListWorkers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListWorkersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ListWorkers(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ListWorkers_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ListWorkers(ctx, req.(*ListWorkersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActorTemplate(ctx, req.(*CreateActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_GetActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).GetActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_GetActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).GetActorTemplate(ctx, req.(*GetActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_DeleteActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).DeleteActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_DeleteActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).DeleteActorTemplate(ctx, req.(*DeleteActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Control_ServiceDesc is the grpc.ServiceDesc for Control service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Control_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "substrate.ateapi.v1.Control", + HandlerType: (*ControlServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetActor", + Handler: _Control_GetActor_Handler, + }, + { + MethodName: "CreateActor", + Handler: _Control_CreateActor_Handler, + }, + { + MethodName: "SuspendActor", + Handler: _Control_SuspendActor_Handler, + }, + { + MethodName: "ResumeActor", + Handler: _Control_ResumeActor_Handler, + }, + { + MethodName: "DeleteActor", + Handler: _Control_DeleteActor_Handler, + }, + { + MethodName: "CreateActorEgressPolicy", + Handler: _Control_CreateActorEgressPolicy_Handler, + }, + { + MethodName: "ListWorkers", + Handler: _Control_ListWorkers_Handler, + }, + { + MethodName: "CreateActorTemplate", + Handler: _Control_CreateActorTemplate_Handler, + }, + { + MethodName: "GetActorTemplate", + Handler: _Control_GetActorTemplate_Handler, + }, + { + MethodName: "DeleteActorTemplate", + Handler: _Control_DeleteActorTemplate_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "control.proto", +} diff --git a/src/crux/sandbox/substrateapi/egress.pb.go b/src/crux/sandbox/substrateapi/egress.pb.go new file mode 100644 index 0000000000..170d873c8a --- /dev/null +++ b/src/crux/sandbox/substrateapi/egress.pb.go @@ -0,0 +1,553 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: egress.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EgressPolicy struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Rules []*EgressRule `protobuf:"bytes,2,rep,name=rules,proto3" json:"rules,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EgressPolicy) Reset() { + *x = EgressPolicy{} + mi := &file_egress_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EgressPolicy) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EgressPolicy) ProtoMessage() {} + +func (x *EgressPolicy) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. +func (*EgressPolicy) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{0} +} + +func (x *EgressPolicy) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *EgressPolicy) GetRules() []*EgressRule { + if x != nil { + return x.Rules + } + return nil +} + +type EgressRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Http *HttpRule `protobuf:"bytes,1,opt,name=http,proto3" json:"http,omitempty"` + Https *HttpsRule `protobuf:"bytes,2,opt,name=https,proto3" json:"https,omitempty"` + TlsPassthrough *TlsPassthroughRule `protobuf:"bytes,3,opt,name=tls_passthrough,json=tlsPassthrough,proto3" json:"tls_passthrough,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EgressRule) Reset() { + *x = EgressRule{} + mi := &file_egress_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EgressRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EgressRule) ProtoMessage() {} + +func (x *EgressRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. +func (*EgressRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{1} +} + +func (x *EgressRule) GetHttp() *HttpRule { + if x != nil { + return x.Http + } + return nil +} + +func (x *EgressRule) GetHttps() *HttpsRule { + if x != nil { + return x.Https + } + return nil +} + +func (x *EgressRule) GetTlsPassthrough() *TlsPassthroughRule { + if x != nil { + return x.TlsPassthrough + } + return nil +} + +type HttpRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpRule) Reset() { + *x = HttpRule{} + mi := &file_egress_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpRule) ProtoMessage() {} + +func (x *HttpRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpRule.ProtoReflect.Descriptor instead. +func (*HttpRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{2} +} + +func (x *HttpRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *HttpRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type HttpsRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpsRule) Reset() { + *x = HttpsRule{} + mi := &file_egress_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpsRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpsRule) ProtoMessage() {} + +func (x *HttpsRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpsRule.ProtoReflect.Descriptor instead. +func (*HttpsRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{3} +} + +func (x *HttpsRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *HttpsRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type TlsPassthroughRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TlsPassthroughRule) Reset() { + *x = TlsPassthroughRule{} + mi := &file_egress_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TlsPassthroughRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TlsPassthroughRule) ProtoMessage() {} + +func (x *TlsPassthroughRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TlsPassthroughRule.ProtoReflect.Descriptor instead. +func (*TlsPassthroughRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{4} +} + +func (x *TlsPassthroughRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *TlsPassthroughRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type Ports struct { + state protoimpl.MessageState `protogen:"open.v1"` + All *AllPorts `protobuf:"bytes,1,opt,name=all,proto3" json:"all,omitempty"` + Numbers []int32 `protobuf:"varint,2,rep,packed,name=numbers,proto3" json:"numbers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ports) Reset() { + *x = Ports{} + mi := &file_egress_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ports) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ports) ProtoMessage() {} + +func (x *Ports) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ports.ProtoReflect.Descriptor instead. +func (*Ports) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{5} +} + +func (x *Ports) GetAll() *AllPorts { + if x != nil { + return x.All + } + return nil +} + +func (x *Ports) GetNumbers() []int32 { + if x != nil { + return x.Numbers + } + return nil +} + +type AllPorts struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AllPorts) Reset() { + *x = AllPorts{} + mi := &file_egress_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AllPorts) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllPorts) ProtoMessage() {} + +func (x *AllPorts) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllPorts.ProtoReflect.Descriptor instead. +func (*AllPorts) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{6} +} + +type CreateActorEgressPolicyRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + EgressPolicy *EgressPolicy `protobuf:"bytes,2,opt,name=egress_policy,json=egressPolicy,proto3" json:"egress_policy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorEgressPolicyRequest) Reset() { + *x = CreateActorEgressPolicyRequest{} + mi := &file_egress_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorEgressPolicyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorEgressPolicyRequest) ProtoMessage() {} + +func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. +func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{7} +} + +func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *CreateActorEgressPolicyRequest) GetEgressPolicy() *EgressPolicy { + if x != nil { + return x.EgressPolicy + } + return nil +} + +var File_egress_proto protoreflect.FileDescriptor + +const file_egress_proto_rawDesc = "" + + "\n" + + "\fegress.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\"\x88\x01\n" + + "\fEgressPolicy\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x125\n" + + "\x05rules\x18\x02 \x03(\v2\x1f.substrate.ateapi.v1.EgressRuleR\x05rules\"\xc7\x01\n" + + "\n" + + "EgressRule\x121\n" + + "\x04http\x18\x01 \x01(\v2\x1d.substrate.ateapi.v1.HttpRuleR\x04http\x124\n" + + "\x05https\x18\x02 \x01(\v2\x1e.substrate.ateapi.v1.HttpsRuleR\x05https\x12P\n" + + "\x0ftls_passthrough\x18\x03 \x01(\v2'.substrate.ateapi.v1.TlsPassthroughRuleR\x0etlsPassthrough\"Z\n" + + "\bHttpRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"[\n" + + "\tHttpsRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"d\n" + + "\x12TlsPassthroughRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"R\n" + + "\x05Ports\x12/\n" + + "\x03all\x18\x01 \x01(\v2\x1d.substrate.ateapi.v1.AllPortsR\x03all\x12\x18\n" + + "\anumbers\x18\x02 \x03(\x05R\anumbers\"\n" + + "\n" + + "\bAllPorts\"\x9e\x01\n" + + "\x1eCreateActorEgressPolicyRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12F\n" + + "\regress_policy\x18\x02 \x01(\v2!.substrate.ateapi.v1.EgressPolicyR\fegressPolicyBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_egress_proto_rawDescOnce sync.Once + file_egress_proto_rawDescData []byte +) + +func file_egress_proto_rawDescGZIP() []byte { + file_egress_proto_rawDescOnce.Do(func() { + file_egress_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_egress_proto_rawDesc), len(file_egress_proto_rawDesc))) + }) + return file_egress_proto_rawDescData +} + +var file_egress_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_egress_proto_goTypes = []any{ + (*EgressPolicy)(nil), // 0: substrate.ateapi.v1.EgressPolicy + (*EgressRule)(nil), // 1: substrate.ateapi.v1.EgressRule + (*HttpRule)(nil), // 2: substrate.ateapi.v1.HttpRule + (*HttpsRule)(nil), // 3: substrate.ateapi.v1.HttpsRule + (*TlsPassthroughRule)(nil), // 4: substrate.ateapi.v1.TlsPassthroughRule + (*Ports)(nil), // 5: substrate.ateapi.v1.Ports + (*AllPorts)(nil), // 6: substrate.ateapi.v1.AllPorts + (*CreateActorEgressPolicyRequest)(nil), // 7: substrate.ateapi.v1.CreateActorEgressPolicyRequest + (*ResourceMetadata)(nil), // 8: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 9: substrate.ateapi.v1.ObjectRef +} +var file_egress_proto_depIdxs = []int32{ + 8, // 0: substrate.ateapi.v1.EgressPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 1, // 1: substrate.ateapi.v1.EgressPolicy.rules:type_name -> substrate.ateapi.v1.EgressRule + 2, // 2: substrate.ateapi.v1.EgressRule.http:type_name -> substrate.ateapi.v1.HttpRule + 3, // 3: substrate.ateapi.v1.EgressRule.https:type_name -> substrate.ateapi.v1.HttpsRule + 4, // 4: substrate.ateapi.v1.EgressRule.tls_passthrough:type_name -> substrate.ateapi.v1.TlsPassthroughRule + 5, // 5: substrate.ateapi.v1.HttpRule.ports:type_name -> substrate.ateapi.v1.Ports + 5, // 6: substrate.ateapi.v1.HttpsRule.ports:type_name -> substrate.ateapi.v1.Ports + 5, // 7: substrate.ateapi.v1.TlsPassthroughRule.ports:type_name -> substrate.ateapi.v1.Ports + 6, // 8: substrate.ateapi.v1.Ports.all:type_name -> substrate.ateapi.v1.AllPorts + 9, // 9: substrate.ateapi.v1.CreateActorEgressPolicyRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 0, // 10: substrate.ateapi.v1.CreateActorEgressPolicyRequest.egress_policy:type_name -> substrate.ateapi.v1.EgressPolicy + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_egress_proto_init() } +func file_egress_proto_init() { + if File_egress_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_egress_proto_rawDesc), len(file_egress_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_egress_proto_goTypes, + DependencyIndexes: file_egress_proto_depIdxs, + MessageInfos: file_egress_proto_msgTypes, + }.Build() + File_egress_proto = out.File + file_egress_proto_goTypes = nil + file_egress_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/egress.proto b/src/crux/sandbox/substrateapi/egress.proto new file mode 100644 index 0000000000..8922f27f97 --- /dev/null +++ b/src/crux/sandbox/substrateapi/egress.proto @@ -0,0 +1,68 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message EgressPolicy { + ResourceMetadata metadata = 1; + repeated EgressRule rules = 2; +} + +message EgressRule { + HttpRule http = 1; + HttpsRule https = 2; + TlsPassthroughRule tls_passthrough = 3; +} + +message HttpRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message HttpsRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message TlsPassthroughRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message Ports { + AllPorts all = 1; + repeated int32 numbers = 2; +} + +message AllPorts {} + +message CreateActorEgressPolicyRequest { + ObjectRef actor = 1; + EgressPolicy egress_policy = 2; +} diff --git a/src/crux/sandbox/substrateapi/generate.sh b/src/crux/sandbox/substrateapi/generate.sh new file mode 100755 index 0000000000..fbfed28814 --- /dev/null +++ b/src/crux/sandbox/substrateapi/generate.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +# Regenerates the *.pb.go files from the trimmed *.proto subset. +# Requires buf, protoc-gen-go (v1.36.x) and protoc-gen-go-grpc on PATH. +set -eu +cd "$(dirname "$0")" +buf generate diff --git a/src/crux/sandbox/substrateapi/substrateapi_test.go b/src/crux/sandbox/substrateapi/substrateapi_test.go new file mode 100644 index 0000000000..68047d96be --- /dev/null +++ b/src/crux/sandbox/substrateapi/substrateapi_test.go @@ -0,0 +1,364 @@ +package substrateapi + +import ( + "context" + "errors" + "net" + "reflect" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" +) + +// The generated code is a wire-compatible subset of Substrate's Control API. +// These tests exercise every message's accessors — empty and populated — and +// every RPC through a real gRPC connection so a regeneration that drops or +// renames a field or method fails here rather than against a cluster. + +// allFiles is a function because the File_* descriptors are populated by +// the generated init functions, after package-level vars. +func allFiles() []protoreflect.FileDescriptor { + return []protoreflect.FileDescriptor{ + File_common_proto, File_actor_proto, File_actor_rpc_proto, File_container_proto, + File_template_proto, File_egress_proto, File_worker_proto, File_control_proto, + } +} + +// callZeroArgMethods invokes every method on v that takes no arguments and +// does not panic on a nil or populated receiver. +func callZeroArgMethods(t *testing.T, v reflect.Value, onlyGetters bool) { + t.Helper() + for i := 0; i < v.NumMethod(); i++ { + m := v.Type().Method(i) + if m.Type.NumIn() != 1 || m.Type.IsVariadic() { + continue + } + if onlyGetters && !strings.HasPrefix(m.Name, "Get") { + continue + } + v.Method(i).Call(nil) + } +} + +// populate sets every field of m to a non-default value, recursing into +// message fields one level so nested getters see data. +func populate(m protoreflect.Message, depth int) { + fields := m.Descriptor().Fields() + for i := 0; i < fields.Len(); i++ { + fd := fields.Get(i) + switch { + case fd.IsList(): + list := m.Mutable(fd).List() + list.Append(sampleValue(fd, list.NewElement, depth)) + case fd.IsMap(): + mp := m.Mutable(fd).Map() + mp.Set(protoreflect.ValueOfString("k").MapKey(), sampleValue(fd.MapValue(), mp.NewValue, depth)) + default: + m.Set(fd, sampleValue(fd, func() protoreflect.Value { return m.NewField(fd) }, depth)) + } + } +} + +func sampleValue(fd protoreflect.FieldDescriptor, newVal func() protoreflect.Value, depth int) protoreflect.Value { + switch fd.Kind() { + case protoreflect.BoolKind: + return protoreflect.ValueOfBool(true) + case protoreflect.StringKind: + return protoreflect.ValueOfString("s") + case protoreflect.BytesKind: + return protoreflect.ValueOfBytes([]byte("b")) + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind: + return protoreflect.ValueOfInt32(7) + case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + return protoreflect.ValueOfInt64(7) + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind: + return protoreflect.ValueOfUint32(7) + case protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + return protoreflect.ValueOfUint64(7) + case protoreflect.FloatKind: + return protoreflect.ValueOfFloat32(1.5) + case protoreflect.DoubleKind: + return protoreflect.ValueOfFloat64(1.5) + case protoreflect.EnumKind: + vals := fd.Enum().Values() + return protoreflect.ValueOfEnum(vals.Get(vals.Len() - 1).Number()) + case protoreflect.MessageKind, protoreflect.GroupKind: + v := newVal() + if depth > 0 { + populate(v.Message(), depth-1) + } + return v + } + return newVal() +} + +func TestEveryMessageAccessorAndRoundTrip(t *testing.T) { + seen := 0 + for _, file := range allFiles() { + msgs := file.Messages() + for i := 0; i < msgs.Len(); i++ { + md := msgs.Get(i) + mt, err := protoregistry.GlobalTypes.FindMessageByName(md.FullName()) + if err != nil { + t.Fatalf("%s not registered: %v", md.FullName(), err) + } + seen++ + // Nil receiver: every getter must return the zero value. + nilMsg := reflect.New(reflect.TypeOf(mt.Zero().Interface()).Elem()) + callZeroArgMethods(t, reflect.Zero(nilMsg.Type()), true) + + // Populated receiver, then a marshal/unmarshal round trip. + full := mt.New() + populate(full, 2) + goVal := reflect.ValueOf(full.Interface()) + callZeroArgMethods(t, goVal, false) + raw, err := proto.Marshal(full.Interface()) + if err != nil { + t.Fatalf("%s marshal: %v", md.FullName(), err) + } + back := mt.New().Interface() + if err := proto.Unmarshal(raw, back); err != nil { + t.Fatalf("%s unmarshal: %v", md.FullName(), err) + } + if !proto.Equal(full.Interface(), back) { + t.Errorf("%s round trip lost data", md.FullName()) + } + // Reset then String on the emptied message. + proto.Reset(back) + callZeroArgMethods(t, reflect.ValueOf(back), false) + } + enums := file.Enums() + for i := 0; i < enums.Len(); i++ { + ed := enums.Get(i) + et, err := protoregistry.GlobalTypes.FindEnumByName(ed.FullName()) + if err != nil { + t.Fatalf("%s not registered: %v", ed.FullName(), err) + } + for j := 0; j < ed.Values().Len(); j++ { + e := et.New(ed.Values().Get(j).Number()) + v := reflect.ValueOf(e) + callZeroArgMethods(t, v, false) + p := reflect.New(v.Type()) + p.Elem().Set(v) + callZeroArgMethods(t, p, false) + if s := v.MethodByName("String").Call(nil)[0].String(); s == "" { + t.Errorf("%s value %d has empty String()", ed.FullName(), j) + } + } + } + } + if seen < 40 { + t.Errorf("only %d messages found; the subset shrank unexpectedly", seen) + } +} + +func TestFieldNumbersMatchUpstream(t *testing.T) { + // A few load-bearing field numbers from Substrate's ateapi/v1 protos; + // renumbering any of these would silently talk past the server. + checks := map[string]map[string]int32{ + "substrate.ateapi.v1.ObjectRef": {"atespace": 1, "name": 2}, + "substrate.ateapi.v1.ResourceMetadata": {"atespace": 1, "name": 2}, + "substrate.ateapi.v1.Actor": {"metadata": 1, "actor_template": 2, "worker_selector": 3, "status": 5}, + "substrate.ateapi.v1.Container": {"name": 1, "image": 2, "command": 3, "env": 5, "volume_mounts": 7}, + } + for name, fields := range checks { + md, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(name)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + m := md.(protoreflect.MessageDescriptor) + for fname, want := range fields { + fd := m.Fields().ByName(protoreflect.Name(fname)) + if fd == nil { + t.Errorf("%s.%s missing", name, fname) + continue + } + if int32(fd.Number()) != want { + t.Errorf("%s.%s = %d, want %d", name, fname, fd.Number(), want) + } + } + } +} + +// fakeServer answers every RPC by echoing the request's target back. +type fakeServer struct { + UnimplementedControlServer + calls []string +} + +func (f *fakeServer) note(name string) { f.calls = append(f.calls, name) } + +func (f *fakeServer) GetActor(_ context.Context, r *GetActorRequest) (*Actor, error) { + f.note("GetActor") + return &Actor{Metadata: &ResourceMetadata{Name: r.GetActor().GetName()}, Status: &ActorStatus{State: ActorState_ACTOR_STATE_RUNNING}}, nil +} +func (f *fakeServer) CreateActor(_ context.Context, r *CreateActorRequest) (*Actor, error) { + f.note("CreateActor") + return r.GetActor(), nil +} +func (f *fakeServer) SuspendActor(_ context.Context, r *SuspendActorRequest) (*SuspendActorResponse, error) { + f.note("SuspendActor:" + r.GetOptions().GetMode().String()) + return &SuspendActorResponse{}, nil +} +func (f *fakeServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { + f.note("ResumeActor") + return &ResumeActorResponse{}, nil +} +func (f *fakeServer) DeleteActor(_ context.Context, r *DeleteActorRequest) (*Actor, error) { + f.note("DeleteActor") + return &Actor{Metadata: &ResourceMetadata{Name: r.GetActor().GetName()}}, nil +} +func (f *fakeServer) CreateActorEgressPolicy(_ context.Context, r *CreateActorEgressPolicyRequest) (*EgressPolicy, error) { + f.note("CreateActorEgressPolicy") + return r.GetEgressPolicy(), nil +} +func (f *fakeServer) ListWorkers(_ context.Context, r *ListWorkersRequest) (*ListWorkersResponse, error) { + f.note("ListWorkers:" + r.GetOptions().GetPageToken()) + if r.GetOptions().GetPageToken() == "" { + return &ListWorkersResponse{Workers: []*Worker{{Metadata: &ResourceMetadata{Name: "w1"}}}, NextPageToken: "p2"}, nil + } + return &ListWorkersResponse{Workers: []*Worker{{Metadata: &ResourceMetadata{Name: "w2"}}}}, nil +} +func (f *fakeServer) CreateActorTemplate(_ context.Context, r *CreateActorTemplateRequest) (*ActorTemplate, error) { + f.note("CreateActorTemplate") + return r.GetActorTemplate(), nil +} +func (f *fakeServer) GetActorTemplate(_ context.Context, r *GetActorTemplateRequest) (*ActorTemplate, error) { + f.note("GetActorTemplate") + return &ActorTemplate{Metadata: &ResourceMetadata{Name: r.GetActorTemplate().GetName()}}, nil +} +func (f *fakeServer) DeleteActorTemplate(_ context.Context, r *DeleteActorTemplateRequest) (*ActorTemplate, error) { + f.note("DeleteActorTemplate") + return &ActorTemplate{Metadata: &ResourceMetadata{Name: r.GetActorTemplate().GetName()}}, nil +} + +func dialBufconn(t *testing.T, srv ControlServer) ControlClient { + t.Helper() + ln := bufconn.Listen(1 << 20) + gs := grpc.NewServer() + RegisterControlServer(gs, srv) + go func() { _ = gs.Serve(ln) }() + t.Cleanup(gs.Stop) + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return ln.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return NewControlClient(conn) +} + +func TestControlClientRoundTripsEveryRPC(t *testing.T) { + srv := &fakeServer{} + c := dialBufconn(t, srv) + ctx := context.Background() + ref := &ObjectRef{Atespace: "hive", Name: "a1"} + + tpl, err := c.CreateActorTemplate(ctx, &CreateActorTemplateRequest{ActorTemplate: &ActorTemplate{Metadata: &ResourceMetadata{Name: "t1"}}}) + if err != nil || tpl.GetMetadata().GetName() != "t1" { + t.Fatalf("CreateActorTemplate: %v %v", tpl, err) + } + if got, err := c.GetActorTemplate(ctx, &GetActorTemplateRequest{ActorTemplate: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("GetActorTemplate: %v %v", got, err) + } + if got, err := c.DeleteActorTemplate(ctx, &DeleteActorTemplateRequest{ActorTemplate: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("DeleteActorTemplate: %v %v", got, err) + } + actor, err := c.CreateActor(ctx, &CreateActorRequest{Actor: &Actor{Metadata: &ResourceMetadata{Name: "a1"}, ActorTemplate: ref}}) + if err != nil || actor.GetActorTemplate().GetName() != "a1" { + t.Fatalf("CreateActor: %v %v", actor, err) + } + if got, err := c.GetActor(ctx, &GetActorRequest{Actor: ref}); err != nil || got.GetStatus().GetState() != ActorState_ACTOR_STATE_RUNNING { + t.Fatalf("GetActor: %v %v", got, err) + } + if _, err := c.ResumeActor(ctx, &ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor: %v", err) + } + if _, err := c.SuspendActor(ctx, &SuspendActorRequest{Actor: ref, Options: &SuspendOptions{Mode: DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor: %v", err) + } + pol, err := c.CreateActorEgressPolicy(ctx, &CreateActorEgressPolicyRequest{Actor: ref, EgressPolicy: &EgressPolicy{Metadata: &ResourceMetadata{Name: "default"}}}) + if err != nil || pol.GetMetadata().GetName() != "default" { + t.Fatalf("CreateActorEgressPolicy: %v %v", pol, err) + } + if got, err := c.DeleteActor(ctx, &DeleteActorRequest{Actor: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("DeleteActor: %v %v", got, err) + } + page1, err := c.ListWorkers(ctx, &ListWorkersRequest{Options: &ListOptions{}}) + if err != nil || page1.GetNextPageToken() != "p2" || len(page1.GetWorkers()) != 1 { + t.Fatalf("ListWorkers page 1: %v %v", page1, err) + } + page2, err := c.ListWorkers(ctx, &ListWorkersRequest{Options: &ListOptions{PageToken: "p2"}}) + if err != nil || page2.GetNextPageToken() != "" || page2.GetWorkers()[0].GetMetadata().GetName() != "w2" { + t.Fatalf("ListWorkers page 2: %v %v", page2, err) + } + joined := strings.Join(srv.calls, ",") + for _, want := range []string{"CreateActorTemplate", "GetActorTemplate", "DeleteActorTemplate", "CreateActor", "GetActor", "ResumeActor", "SuspendActor:DURABILITY_MODE_ASYNCHRONOUS", "CreateActorEgressPolicy", "DeleteActor", "ListWorkers:", "ListWorkers:p2"} { + if !strings.Contains(joined, want) { + t.Errorf("server never saw %s: %s", want, joined) + } + } +} + +func TestUnimplementedServerAnswersUnimplemented(t *testing.T) { + c := dialBufconn(t, UnimplementedControlServer{}) + ctx := context.Background() + calls := []func() error{ + func() error { _, err := c.GetActor(ctx, &GetActorRequest{}); return err }, + func() error { _, err := c.CreateActor(ctx, &CreateActorRequest{}); return err }, + func() error { _, err := c.SuspendActor(ctx, &SuspendActorRequest{}); return err }, + func() error { _, err := c.ResumeActor(ctx, &ResumeActorRequest{}); return err }, + func() error { _, err := c.DeleteActor(ctx, &DeleteActorRequest{}); return err }, + func() error { _, err := c.CreateActorEgressPolicy(ctx, &CreateActorEgressPolicyRequest{}); return err }, + func() error { _, err := c.ListWorkers(ctx, &ListWorkersRequest{}); return err }, + func() error { _, err := c.CreateActorTemplate(ctx, &CreateActorTemplateRequest{}); return err }, + func() error { _, err := c.GetActorTemplate(ctx, &GetActorTemplateRequest{}); return err }, + func() error { _, err := c.DeleteActorTemplate(ctx, &DeleteActorTemplateRequest{}); return err }, + } + for i, call := range calls { + err := call() + if st, ok := status.FromError(err); !ok || st.Code() != codes.Unimplemented { + t.Errorf("call %d: %v, want Unimplemented", i, err) + } + } + var u UnimplementedControlServer + u.testEmbeddedByValue() + if _, err := u.GetActor(ctx, nil); err == nil { + t.Error("direct GetActor on the unimplemented server returned nil error") + } + var _ UnsafeControlServer = &fakeServer{} +} + +func TestRegisterControlServerRejectsNilEmbedding(t *testing.T) { + // Registering a server whose embedded UnimplementedControlServer is a + // nil pointer must panic, which is grpc-go's guard against a forgotten + // embedding. + defer func() { + if r := recover(); r == nil { + t.Error("no panic for a nil embedded UnimplementedControlServer") + } + }() + type nilEmbed struct{ *UnimplementedControlServer } + RegisterControlServer(grpc.NewServer(), nilEmbed{}) +} + +func TestServiceDescriptor(t *testing.T) { + if Control_ServiceDesc.ServiceName != "substrate.ateapi.v1.Control" { + t.Errorf("service name = %q", Control_ServiceDesc.ServiceName) + } + if len(Control_ServiceDesc.Methods) != 10 { + t.Errorf("methods = %d, want 10", len(Control_ServiceDesc.Methods)) + } + if !errors.Is(context.Canceled, context.Canceled) { + t.Fatal("unreachable") + } +} diff --git a/src/crux/sandbox/substrateapi/template.pb.go b/src/crux/sandbox/substrateapi/template.pb.go new file mode 100644 index 0000000000..b27724d605 --- /dev/null +++ b/src/crux/sandbox/substrateapi/template.pb.go @@ -0,0 +1,616 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: template.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GoldenSnapshotMode int32 + +const ( + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_UNSPECIFIED GoldenSnapshotMode = 0 + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_ENABLED GoldenSnapshotMode = 1 + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED GoldenSnapshotMode = 2 +) + +// Enum value maps for GoldenSnapshotMode. +var ( + GoldenSnapshotMode_name = map[int32]string{ + 0: "GOLDEN_SNAPSHOT_MODE_UNSPECIFIED", + 1: "GOLDEN_SNAPSHOT_MODE_ENABLED", + 2: "GOLDEN_SNAPSHOT_MODE_DISABLED", + } + GoldenSnapshotMode_value = map[string]int32{ + "GOLDEN_SNAPSHOT_MODE_UNSPECIFIED": 0, + "GOLDEN_SNAPSHOT_MODE_ENABLED": 1, + "GOLDEN_SNAPSHOT_MODE_DISABLED": 2, + } +) + +func (x GoldenSnapshotMode) Enum() *GoldenSnapshotMode { + p := new(GoldenSnapshotMode) + *p = x + return p +} + +func (x GoldenSnapshotMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GoldenSnapshotMode) Descriptor() protoreflect.EnumDescriptor { + return file_template_proto_enumTypes[0].Descriptor() +} + +func (GoldenSnapshotMode) Type() protoreflect.EnumType { + return &file_template_proto_enumTypes[0] +} + +func (x GoldenSnapshotMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GoldenSnapshotMode.Descriptor instead. +func (GoldenSnapshotMode) EnumDescriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{0} +} + +type ActorTemplate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + WorkerSelector *Selector `protobuf:"bytes,2,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` + Containers []*Container `protobuf:"bytes,3,rep,name=containers,proto3" json:"containers,omitempty"` + Volumes []*Volume `protobuf:"bytes,4,rep,name=volumes,proto3" json:"volumes,omitempty"` + SnapshotConfig *SnapshotConfig `protobuf:"bytes,5,opt,name=snapshot_config,json=snapshotConfig,proto3" json:"snapshot_config,omitempty"` + Resources *ResourceRequirements `protobuf:"bytes,6,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorTemplate) Reset() { + *x = ActorTemplate{} + mi := &file_template_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorTemplate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorTemplate) ProtoMessage() {} + +func (x *ActorTemplate) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorTemplate.ProtoReflect.Descriptor instead. +func (*ActorTemplate) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{0} +} + +func (x *ActorTemplate) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ActorTemplate) GetWorkerSelector() *Selector { + if x != nil { + return x.WorkerSelector + } + return nil +} + +func (x *ActorTemplate) GetContainers() []*Container { + if x != nil { + return x.Containers + } + return nil +} + +func (x *ActorTemplate) GetVolumes() []*Volume { + if x != nil { + return x.Volumes + } + return nil +} + +func (x *ActorTemplate) GetSnapshotConfig() *SnapshotConfig { + if x != nil { + return x.SnapshotConfig + } + return nil +} + +func (x *ActorTemplate) GetResources() *ResourceRequirements { + if x != nil { + return x.Resources + } + return nil +} + +type ResourceRequirements struct { + state protoimpl.MessageState `protogen:"open.v1"` + Limits *Resources `protobuf:"bytes,1,opt,name=limits,proto3" json:"limits,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceRequirements) Reset() { + *x = ResourceRequirements{} + mi := &file_template_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceRequirements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceRequirements) ProtoMessage() {} + +func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceRequirements.ProtoReflect.Descriptor instead. +func (*ResourceRequirements) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{1} +} + +func (x *ResourceRequirements) GetLimits() *Resources { + if x != nil { + return x.Limits + } + return nil +} + +type Resources struct { + state protoimpl.MessageState `protogen:"open.v1"` + CpuMillicores int64 `protobuf:"varint,1,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` + MemoryBytes int64 `protobuf:"varint,2,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Resources) Reset() { + *x = Resources{} + mi := &file_template_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Resources) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Resources) ProtoMessage() {} + +func (x *Resources) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Resources.ProtoReflect.Descriptor instead. +func (*Resources) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{2} +} + +func (x *Resources) GetCpuMillicores() int64 { + if x != nil { + return x.CpuMillicores + } + return 0 +} + +func (x *Resources) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +type SnapshotConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + GoldenSnapshotConfig *GoldenSnapshotConfig `protobuf:"bytes,3,opt,name=golden_snapshot_config,json=goldenSnapshotConfig,proto3" json:"golden_snapshot_config,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotConfig) Reset() { + *x = SnapshotConfig{} + mi := &file_template_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotConfig) ProtoMessage() {} + +func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotConfig.ProtoReflect.Descriptor instead. +func (*SnapshotConfig) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{3} +} + +func (x *SnapshotConfig) GetGoldenSnapshotConfig() *GoldenSnapshotConfig { + if x != nil { + return x.GoldenSnapshotConfig + } + return nil +} + +type GoldenSnapshotConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode GoldenSnapshotMode `protobuf:"varint,1,opt,name=mode,proto3,enum=substrate.ateapi.v1.GoldenSnapshotMode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GoldenSnapshotConfig) Reset() { + *x = GoldenSnapshotConfig{} + mi := &file_template_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GoldenSnapshotConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GoldenSnapshotConfig) ProtoMessage() {} + +func (x *GoldenSnapshotConfig) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GoldenSnapshotConfig.ProtoReflect.Descriptor instead. +func (*GoldenSnapshotConfig) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{4} +} + +func (x *GoldenSnapshotConfig) GetMode() GoldenSnapshotMode { + if x != nil { + return x.Mode + } + return GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_UNSPECIFIED +} + +type CreateActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ActorTemplate `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorTemplateRequest) Reset() { + *x = CreateActorTemplateRequest{} + mi := &file_template_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorTemplateRequest) ProtoMessage() {} + +func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*CreateActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{5} +} + +func (x *CreateActorTemplateRequest) GetActorTemplate() *ActorTemplate { + if x != nil { + return x.ActorTemplate + } + return nil +} + +type GetActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetActorTemplateRequest) Reset() { + *x = GetActorTemplateRequest{} + mi := &file_template_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetActorTemplateRequest) ProtoMessage() {} + +func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*GetActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{6} +} + +func (x *GetActorTemplateRequest) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +type DeleteActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteActorTemplateRequest) Reset() { + *x = DeleteActorTemplateRequest{} + mi := &file_template_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteActorTemplateRequest) ProtoMessage() {} + +func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*DeleteActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{7} +} + +func (x *DeleteActorTemplateRequest) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +func (x *DeleteActorTemplateRequest) GetOptions() *DeleteOptions { + if x != nil { + return x.Options + } + return nil +} + +var File_template_proto protoreflect.FileDescriptor + +const file_template_proto_rawDesc = "" + + "\n" + + "\x0etemplate.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\x1a\x0fcontainer.proto\"\xa8\x03\n" + + "\rActorTemplate\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12F\n" + + "\x0fworker_selector\x18\x02 \x01(\v2\x1d.substrate.ateapi.v1.SelectorR\x0eworkerSelector\x12>\n" + + "\n" + + "containers\x18\x03 \x03(\v2\x1e.substrate.ateapi.v1.ContainerR\n" + + "containers\x125\n" + + "\avolumes\x18\x04 \x03(\v2\x1b.substrate.ateapi.v1.VolumeR\avolumes\x12L\n" + + "\x0fsnapshot_config\x18\x05 \x01(\v2#.substrate.ateapi.v1.SnapshotConfigR\x0esnapshotConfig\x12G\n" + + "\tresources\x18\x06 \x01(\v2).substrate.ateapi.v1.ResourceRequirementsR\tresources\"N\n" + + "\x14ResourceRequirements\x126\n" + + "\x06limits\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ResourcesR\x06limits\"U\n" + + "\tResources\x12%\n" + + "\x0ecpu_millicores\x18\x01 \x01(\x03R\rcpuMillicores\x12!\n" + + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\"q\n" + + "\x0eSnapshotConfig\x12_\n" + + "\x16golden_snapshot_config\x18\x03 \x01(\v2).substrate.ateapi.v1.GoldenSnapshotConfigR\x14goldenSnapshotConfig\"S\n" + + "\x14GoldenSnapshotConfig\x12;\n" + + "\x04mode\x18\x01 \x01(\x0e2'.substrate.ateapi.v1.GoldenSnapshotModeR\x04mode\"g\n" + + "\x1aCreateActorTemplateRequest\x12I\n" + + "\x0eactor_template\x18\x01 \x01(\v2\".substrate.ateapi.v1.ActorTemplateR\ractorTemplate\"`\n" + + "\x17GetActorTemplateRequest\x12E\n" + + "\x0eactor_template\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\"\xa1\x01\n" + + "\x1aDeleteActorTemplateRequest\x12E\n" + + "\x0eactor_template\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\x12<\n" + + "\aoptions\x18\x02 \x01(\v2\".substrate.ateapi.v1.DeleteOptionsR\aoptions*\x7f\n" + + "\x12GoldenSnapshotMode\x12$\n" + + " GOLDEN_SNAPSHOT_MODE_UNSPECIFIED\x10\x00\x12 \n" + + "\x1cGOLDEN_SNAPSHOT_MODE_ENABLED\x10\x01\x12!\n" + + "\x1dGOLDEN_SNAPSHOT_MODE_DISABLED\x10\x02BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_template_proto_rawDescOnce sync.Once + file_template_proto_rawDescData []byte +) + +func file_template_proto_rawDescGZIP() []byte { + file_template_proto_rawDescOnce.Do(func() { + file_template_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_template_proto_rawDesc), len(file_template_proto_rawDesc))) + }) + return file_template_proto_rawDescData +} + +var file_template_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_template_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_template_proto_goTypes = []any{ + (GoldenSnapshotMode)(0), // 0: substrate.ateapi.v1.GoldenSnapshotMode + (*ActorTemplate)(nil), // 1: substrate.ateapi.v1.ActorTemplate + (*ResourceRequirements)(nil), // 2: substrate.ateapi.v1.ResourceRequirements + (*Resources)(nil), // 3: substrate.ateapi.v1.Resources + (*SnapshotConfig)(nil), // 4: substrate.ateapi.v1.SnapshotConfig + (*GoldenSnapshotConfig)(nil), // 5: substrate.ateapi.v1.GoldenSnapshotConfig + (*CreateActorTemplateRequest)(nil), // 6: substrate.ateapi.v1.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 7: substrate.ateapi.v1.GetActorTemplateRequest + (*DeleteActorTemplateRequest)(nil), // 8: substrate.ateapi.v1.DeleteActorTemplateRequest + (*ResourceMetadata)(nil), // 9: substrate.ateapi.v1.ResourceMetadata + (*Selector)(nil), // 10: substrate.ateapi.v1.Selector + (*Container)(nil), // 11: substrate.ateapi.v1.Container + (*Volume)(nil), // 12: substrate.ateapi.v1.Volume + (*ObjectRef)(nil), // 13: substrate.ateapi.v1.ObjectRef + (*DeleteOptions)(nil), // 14: substrate.ateapi.v1.DeleteOptions +} +var file_template_proto_depIdxs = []int32{ + 9, // 0: substrate.ateapi.v1.ActorTemplate.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 10, // 1: substrate.ateapi.v1.ActorTemplate.worker_selector:type_name -> substrate.ateapi.v1.Selector + 11, // 2: substrate.ateapi.v1.ActorTemplate.containers:type_name -> substrate.ateapi.v1.Container + 12, // 3: substrate.ateapi.v1.ActorTemplate.volumes:type_name -> substrate.ateapi.v1.Volume + 4, // 4: substrate.ateapi.v1.ActorTemplate.snapshot_config:type_name -> substrate.ateapi.v1.SnapshotConfig + 2, // 5: substrate.ateapi.v1.ActorTemplate.resources:type_name -> substrate.ateapi.v1.ResourceRequirements + 3, // 6: substrate.ateapi.v1.ResourceRequirements.limits:type_name -> substrate.ateapi.v1.Resources + 5, // 7: substrate.ateapi.v1.SnapshotConfig.golden_snapshot_config:type_name -> substrate.ateapi.v1.GoldenSnapshotConfig + 0, // 8: substrate.ateapi.v1.GoldenSnapshotConfig.mode:type_name -> substrate.ateapi.v1.GoldenSnapshotMode + 1, // 9: substrate.ateapi.v1.CreateActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ActorTemplate + 13, // 10: substrate.ateapi.v1.GetActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 13, // 11: substrate.ateapi.v1.DeleteActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 14, // 12: substrate.ateapi.v1.DeleteActorTemplateRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_template_proto_init() } +func file_template_proto_init() { + if File_template_proto != nil { + return + } + file_common_proto_init() + file_container_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_template_proto_rawDesc), len(file_template_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_template_proto_goTypes, + DependencyIndexes: file_template_proto_depIdxs, + EnumInfos: file_template_proto_enumTypes, + MessageInfos: file_template_proto_msgTypes, + }.Build() + File_template_proto = out.File + file_template_proto_goTypes = nil + file_template_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/template.proto b/src/crux/sandbox/substrateapi/template.proto new file mode 100644 index 0000000000..adcde1ee59 --- /dev/null +++ b/src/crux/sandbox/substrateapi/template.proto @@ -0,0 +1,76 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; +import "container.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message ActorTemplate { + ResourceMetadata metadata = 1; + Selector worker_selector = 2; + repeated Container containers = 3; + repeated Volume volumes = 4; + SnapshotConfig snapshot_config = 5; + ResourceRequirements resources = 6; +} + +message ResourceRequirements { + Resources limits = 1; +} + +message Resources { + int64 cpu_millicores = 1; + int64 memory_bytes = 2; +} + +message SnapshotConfig { + GoldenSnapshotConfig golden_snapshot_config = 3; +} + +enum GoldenSnapshotMode { + GOLDEN_SNAPSHOT_MODE_UNSPECIFIED = 0; + GOLDEN_SNAPSHOT_MODE_ENABLED = 1; + GOLDEN_SNAPSHOT_MODE_DISABLED = 2; +} + +message GoldenSnapshotConfig { + GoldenSnapshotMode mode = 1; +} + +message CreateActorTemplateRequest { + ActorTemplate actor_template = 1; +} + +message GetActorTemplateRequest { + ObjectRef actor_template = 1; +} + +message DeleteActorTemplateRequest { + ObjectRef actor_template = 1; + DeleteOptions options = 2; +} diff --git a/src/crux/sandbox/substrateapi/worker.pb.go b/src/crux/sandbox/substrateapi/worker.pb.go new file mode 100644 index 0000000000..07611b1abc --- /dev/null +++ b/src/crux/sandbox/substrateapi/worker.pb.go @@ -0,0 +1,499 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: worker.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type WorkerState int32 + +const ( + WorkerState_WORKER_STATE_UNSPECIFIED WorkerState = 0 + WorkerState_WORKER_STATE_ACTIVE WorkerState = 1 + WorkerState_WORKER_STATE_DRAINING WorkerState = 2 + WorkerState_WORKER_STATE_UNAVAILABLE WorkerState = 3 +) + +// Enum value maps for WorkerState. +var ( + WorkerState_name = map[int32]string{ + 0: "WORKER_STATE_UNSPECIFIED", + 1: "WORKER_STATE_ACTIVE", + 2: "WORKER_STATE_DRAINING", + 3: "WORKER_STATE_UNAVAILABLE", + } + WorkerState_value = map[string]int32{ + "WORKER_STATE_UNSPECIFIED": 0, + "WORKER_STATE_ACTIVE": 1, + "WORKER_STATE_DRAINING": 2, + "WORKER_STATE_UNAVAILABLE": 3, + } +) + +func (x WorkerState) Enum() *WorkerState { + p := new(WorkerState) + *p = x + return p +} + +func (x WorkerState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (WorkerState) Descriptor() protoreflect.EnumDescriptor { + return file_worker_proto_enumTypes[0].Descriptor() +} + +func (WorkerState) Type() protoreflect.EnumType { + return &file_worker_proto_enumTypes[0] +} + +func (x WorkerState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use WorkerState.Descriptor instead. +func (WorkerState) EnumDescriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{0} +} + +type Worker struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + WorkerNamespace string `protobuf:"bytes,2,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` + WorkerPool string `protobuf:"bytes,3,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` + WorkerPod string `protobuf:"bytes,4,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` + WorkerPodUid string `protobuf:"bytes,5,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` + NodeName string `protobuf:"bytes,6,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,7,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Status *WorkerStatus `protobuf:"bytes,9,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Worker) Reset() { + *x = Worker{} + mi := &file_worker_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Worker) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Worker) ProtoMessage() {} + +func (x *Worker) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Worker.ProtoReflect.Descriptor instead. +func (*Worker) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{0} +} + +func (x *Worker) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Worker) GetWorkerNamespace() string { + if x != nil { + return x.WorkerNamespace + } + return "" +} + +func (x *Worker) GetWorkerPool() string { + if x != nil { + return x.WorkerPool + } + return "" +} + +func (x *Worker) GetWorkerPod() string { + if x != nil { + return x.WorkerPod + } + return "" +} + +func (x *Worker) GetWorkerPodUid() string { + if x != nil { + return x.WorkerPodUid + } + return "" +} + +func (x *Worker) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +func (x *Worker) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + +func (x *Worker) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +func (x *Worker) GetStatus() *WorkerStatus { + if x != nil { + return x.Status + } + return nil +} + +type WorkerStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State WorkerState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.WorkerState" json:"state,omitempty"` + SandboxClasses []*WorkerSandboxClass `protobuf:"bytes,5,rep,name=sandbox_classes,json=sandboxClasses,proto3" json:"sandbox_classes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerStatus) Reset() { + *x = WorkerStatus{} + mi := &file_worker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerStatus) ProtoMessage() {} + +func (x *WorkerStatus) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. +func (*WorkerStatus) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{1} +} + +func (x *WorkerStatus) GetState() WorkerState { + if x != nil { + return x.State + } + return WorkerState_WORKER_STATE_UNSPECIFIED +} + +func (x *WorkerStatus) GetSandboxClasses() []*WorkerSandboxClass { + if x != nil { + return x.SandboxClasses + } + return nil +} + +type WorkerSandboxClass struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerSandboxClass) Reset() { + *x = WorkerSandboxClass{} + mi := &file_worker_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerSandboxClass) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerSandboxClass) ProtoMessage() {} + +func (x *WorkerSandboxClass) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerSandboxClass.ProtoReflect.Descriptor instead. +func (*WorkerSandboxClass) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{2} +} + +func (x *WorkerSandboxClass) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +type ListWorkersRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Options *ListOptions `protobuf:"bytes,1,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkersRequest) Reset() { + *x = ListWorkersRequest{} + mi := &file_worker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkersRequest) ProtoMessage() {} + +func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. +func (*ListWorkersRequest) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{3} +} + +func (x *ListWorkersRequest) GetOptions() *ListOptions { + if x != nil { + return x.Options + } + return nil +} + +type ListWorkersResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Workers []*Worker `protobuf:"bytes,1,rep,name=workers,proto3" json:"workers,omitempty"` + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkersResponse) Reset() { + *x = ListWorkersResponse{} + mi := &file_worker_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkersResponse) ProtoMessage() {} + +func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. +func (*ListWorkersResponse) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{4} +} + +func (x *ListWorkersResponse) GetWorkers() []*Worker { + if x != nil { + return x.Workers + } + return nil +} + +func (x *ListWorkersResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + +var File_worker_proto protoreflect.FileDescriptor + +const file_worker_proto_rawDesc = "" + + "\n" + + "\fworker.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\"\xda\x03\n" + + "\x06Worker\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12)\n" + + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + + "\vworker_pool\x18\x03 \x01(\tR\n" + + "workerPool\x12\x1d\n" + + "\n" + + "worker_pod\x18\x04 \x01(\tR\tworkerPod\x12$\n" + + "\x0eworker_pod_uid\x18\x05 \x01(\tR\fworkerPodUid\x12\x1b\n" + + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12(\n" + + "\x10worker_spiffe_id\x18\a \x01(\tR\x0eworkerSpiffeId\x12?\n" + + "\x06labels\x18\b \x03(\v2'.substrate.ateapi.v1.Worker.LabelsEntryR\x06labels\x129\n" + + "\x06status\x18\t \x01(\v2!.substrate.ateapi.v1.WorkerStatusR\x06status\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x98\x01\n" + + "\fWorkerStatus\x126\n" + + "\x05state\x18\x01 \x01(\x0e2 .substrate.ateapi.v1.WorkerStateR\x05state\x12P\n" + + "\x0fsandbox_classes\x18\x05 \x03(\v2'.substrate.ateapi.v1.WorkerSandboxClassR\x0esandboxClasses\"9\n" + + "\x12WorkerSandboxClass\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\"P\n" + + "\x12ListWorkersRequest\x12:\n" + + "\aoptions\x18\x01 \x01(\v2 .substrate.ateapi.v1.ListOptionsR\aoptions\"t\n" + + "\x13ListWorkersResponse\x125\n" + + "\aworkers\x18\x01 \x03(\v2\x1b.substrate.ateapi.v1.WorkerR\aworkers\x12&\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken*}\n" + + "\vWorkerState\x12\x1c\n" + + "\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n" + + "\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n" + + "\x15WORKER_STATE_DRAINING\x10\x02\x12\x1c\n" + + "\x18WORKER_STATE_UNAVAILABLE\x10\x03BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_worker_proto_rawDescOnce sync.Once + file_worker_proto_rawDescData []byte +) + +func file_worker_proto_rawDescGZIP() []byte { + file_worker_proto_rawDescOnce.Do(func() { + file_worker_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_worker_proto_rawDesc), len(file_worker_proto_rawDesc))) + }) + return file_worker_proto_rawDescData +} + +var file_worker_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_worker_proto_goTypes = []any{ + (WorkerState)(0), // 0: substrate.ateapi.v1.WorkerState + (*Worker)(nil), // 1: substrate.ateapi.v1.Worker + (*WorkerStatus)(nil), // 2: substrate.ateapi.v1.WorkerStatus + (*WorkerSandboxClass)(nil), // 3: substrate.ateapi.v1.WorkerSandboxClass + (*ListWorkersRequest)(nil), // 4: substrate.ateapi.v1.ListWorkersRequest + (*ListWorkersResponse)(nil), // 5: substrate.ateapi.v1.ListWorkersResponse + nil, // 6: substrate.ateapi.v1.Worker.LabelsEntry + (*ResourceMetadata)(nil), // 7: substrate.ateapi.v1.ResourceMetadata + (*ListOptions)(nil), // 8: substrate.ateapi.v1.ListOptions +} +var file_worker_proto_depIdxs = []int32{ + 7, // 0: substrate.ateapi.v1.Worker.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 6, // 1: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry + 2, // 2: substrate.ateapi.v1.Worker.status:type_name -> substrate.ateapi.v1.WorkerStatus + 0, // 3: substrate.ateapi.v1.WorkerStatus.state:type_name -> substrate.ateapi.v1.WorkerState + 3, // 4: substrate.ateapi.v1.WorkerStatus.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass + 8, // 5: substrate.ateapi.v1.ListWorkersRequest.options:type_name -> substrate.ateapi.v1.ListOptions + 1, // 6: substrate.ateapi.v1.ListWorkersResponse.workers:type_name -> substrate.ateapi.v1.Worker + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_worker_proto_init() } +func file_worker_proto_init() { + if File_worker_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_worker_proto_rawDesc), len(file_worker_proto_rawDesc)), + NumEnums: 1, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_worker_proto_goTypes, + DependencyIndexes: file_worker_proto_depIdxs, + EnumInfos: file_worker_proto_enumTypes, + MessageInfos: file_worker_proto_msgTypes, + }.Build() + File_worker_proto = out.File + file_worker_proto_goTypes = nil + file_worker_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/worker.proto b/src/crux/sandbox/substrateapi/worker.proto new file mode 100644 index 0000000000..13a527149e --- /dev/null +++ b/src/crux/sandbox/substrateapi/worker.proto @@ -0,0 +1,67 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Worker { + ResourceMetadata metadata = 1; + string worker_namespace = 2; + string worker_pool = 3; + string worker_pod = 4; + string worker_pod_uid = 5; + string node_name = 6; + string worker_spiffe_id = 7; + map labels = 8; + WorkerStatus status = 9; +} + +enum WorkerState { + WORKER_STATE_UNSPECIFIED = 0; + WORKER_STATE_ACTIVE = 1; + WORKER_STATE_DRAINING = 2; + WORKER_STATE_UNAVAILABLE = 3; +} + +message WorkerStatus { + WorkerState state = 1; + repeated WorkerSandboxClass sandbox_classes = 5; +} + +message WorkerSandboxClass { + string sandbox_class = 1; +} + +message ListWorkersRequest { + ListOptions options = 1; +} + +message ListWorkersResponse { + repeated Worker workers = 1; + string next_page_token = 2; +} diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go new file mode 100644 index 0000000000..389ffabb18 --- /dev/null +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -0,0 +1,338 @@ +// Package substratebridge is the in-actor side of hive's Agent Substrate +// runtime (ADR-0026 phase 2). Substrate has no exec, attach or logs RPC and +// cannot mount an existing PersistentVolumeClaim, so the sandbox image runs +// this small HTTP server as the actor's entrypoint and hive drives the kick +// through it over the atenet-router mTLS CONNECT tunnel: push the workspace +// in, run the command, pull the workspace back out. The server listens on +// the actor's private network only; the router is the only way in and it +// admits only allowlisted SPIFFE identities. +package substratebridge + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + // DefaultPort is the port the bridge serves on inside the actor and the + // port hive's CONNECT tunnel targets. + DefaultPort = 8477 + // DefaultWorkspace is where the bridge unpacks the kick workspace. + DefaultWorkspace = "/workspace" + + PathHealth = "/healthz" + PathWorkspace = "/workspace" + PathRun = "/run" + + // MaxOutput caps each of stdout and stderr in a RunResponse; the same + // 16 MiB the OpenShell launcher keeps. + MaxOutput = 16 << 20 + + // ExitCodeUnknown is reported when the command could not be started or + // ended without an exit status. + ExitCodeUnknown = -1 + + contentTypeTar = "application/x-tar+gzip" + contentTypeJSON = "application/json" + maxRequestJSON = 1 << 20 + maxTarEntry = 1 << 30 +) + +// RunRequest is the body of POST /run. +type RunRequest struct { + Command []string `json:"command"` + Env map[string]string `json:"env,omitempty"` + WorkDir string `json:"workdir,omitempty"` +} + +// RunResponse is the body POST /run answers with once the command exits. +type RunResponse struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ExitCode int `json:"exit_code"` + Error string `json:"error,omitempty"` +} + +// Handler serves the bridge protocol for one workspace directory. +type Handler struct { + // Workspace is the directory PUT /workspace fills, GET /workspace packs + // and POST /run runs in. Defaults to DefaultWorkspace. + Workspace string + // RunTimeout bounds one POST /run; zero means the request context alone. + RunTimeout time.Duration +} + +// ServeHTTP implements http.Handler. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == PathHealth && r.Method == http.MethodGet: + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "ok\n") + case r.URL.Path == PathWorkspace && r.Method == http.MethodPut: + h.putWorkspace(w, r) + case r.URL.Path == PathWorkspace && r.Method == http.MethodGet: + h.getWorkspace(w, r) + case r.URL.Path == PathRun && r.Method == http.MethodPost: + h.run(w, r) + default: + http.Error(w, "substrate bridge: unknown route", http.StatusNotFound) + } +} + +func (h *Handler) workspace() string { + if h.Workspace != "" { + return h.Workspace + } + return DefaultWorkspace +} + +func (h *Handler) putWorkspace(w http.ResponseWriter, r *http.Request) { + dir := h.workspace() + if err := os.RemoveAll(dir); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: clearing %s: %v", dir, err), http.StatusInternalServerError) + return + } + if err := Unpack(r.Body, dir); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: unpacking workspace: %v", err), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) getWorkspace(w http.ResponseWriter, _ *http.Request) { + var buf bytes.Buffer + if err := Pack(h.workspace(), &buf); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: packing workspace: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", contentTypeTar) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, &buf) +} + +func (h *Handler) run(w http.ResponseWriter, r *http.Request) { + var req RunRequest + if err := json.NewDecoder(io.LimitReader(r.Body, maxRequestJSON)).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: decoding run request: %v", err), http.StatusBadRequest) + return + } + if len(req.Command) == 0 { + http.Error(w, "substrate bridge: command is required", http.StatusBadRequest) + return + } + ctx := r.Context() + if h.RunTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, h.RunTimeout) + defer cancel() + } + resp := Run(ctx, req, h.workspace()) + w.Header().Set("Content-Type", contentTypeJSON) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) +} + +// Run executes one RunRequest with the request env replacing the process +// env — the launcher already filtered it through hive's credential-free +// allowlist — and reports the exit code the way the Podman launcher does. +func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { + cmd := exec.CommandContext(ctx, req.Command[0], req.Command[1:]...) + cmd.Dir = req.WorkDir + if cmd.Dir == "" { + cmd.Dir = workspace + } + keys := make([]string, 0, len(req.Env)) + for k := range req.Env { + keys = append(keys, k) + } + sort.Strings(keys) + env := make([]string, 0, len(keys)+1) + for _, k := range keys { + env = append(env, k+"="+req.Env[k]) + } + if _, ok := req.Env["PATH"]; !ok { + env = append(env, "PATH="+os.Getenv("PATH")) + } + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + resp := RunResponse{Stdout: capString(stdout.Bytes()), Stderr: capString(stderr.Bytes())} + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + resp.ExitCode = exitErr.ExitCode() + default: + resp.ExitCode = ExitCodeUnknown + resp.Error = err.Error() + } + if ctx.Err() != nil && resp.Error == "" { + resp.Error = ctx.Err().Error() + } + return resp +} + +func capString(b []byte) string { + if len(b) > MaxOutput { + b = b[:MaxOutput] + } + return string(b) +} + +// Pack writes dir as a gzip-compressed tar stream: regular files, directories +// and symlinks, paths relative to dir. Git workspaces are small enough that a +// single stream is simpler than any chunked protocol. +func Pack(dir string, w io.Writer) error { + gz := gzip.NewWriter(w) + tw := tar.NewWriter(gz) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + link := "" + if info.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(rel) + if info.IsDir() { + hdr.Name += "/" + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + return err + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + return gz.Close() +} + +// Unpack reads a Pack stream into dir, refusing entries that would escape it. +func Unpack(r io.Reader, dir string) error { + gz, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip: %w", err) + } + defer gz.Close() + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("tar: %w", err) + } + target, err := safeJoin(dir, hdr.Name) + if err != nil { + return err + } + if err := unpackEntry(tr, hdr, target); err != nil { + return err + } + } +} + +func unpackEntry(tr *tar.Reader, hdr *tar.Header, target string) error { + switch hdr.Typeflag { + case tar.TypeDir: + return os.MkdirAll(target, os.FileMode(hdr.Mode)&os.ModePerm|0o700) + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + _ = os.Remove(target) + return os.Symlink(hdr.Linkname, target) + case tar.TypeReg: + if hdr.Size > maxTarEntry { + return fmt.Errorf("tar: entry %q is too large (%d bytes)", hdr.Name, hdr.Size) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) + if err != nil { + return err + } + if _, err := io.Copy(f, io.LimitReader(tr, hdr.Size)); err != nil { + f.Close() + return err + } + return f.Close() + default: + // Devices, FIFOs and hard links have no place in a git workspace. + return fmt.Errorf("tar: entry %q has unsupported type %q", hdr.Name, hdr.Typeflag) + } +} + +func safeJoin(dir, name string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(name)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("tar: entry %q escapes the workspace", name) + } + return filepath.Join(dir, clean), nil +} + +// Serve runs the bridge on ln until ctx ends. The actor's command is +// expected to be the sandbox image's bridge entrypoint calling this. +func Serve(ctx context.Context, ln net.Listener, h *Handler) error { + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + errCh := make(chan error, 1) + go func() { errCh <- srv.Serve(ln) }() + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutdownCtx) + return ctx.Err() + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go new file mode 100644 index 0000000000..e91754700a --- /dev/null +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -0,0 +1,394 @@ +package substratebridge + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestPackUnpackRoundTrip(t *testing.T) { + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "alpha") + writeFile(t, filepath.Join(src, "sub", "deep", "b.txt"), "beta") + if err := os.Mkdir(filepath.Join(src, "empty"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("a.txt", filepath.Join(src, "link")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(src, "a.txt"), 0o755); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := Pack(src, &buf); err != nil { + t.Fatalf("Pack: %v", err) + } + dst := filepath.Join(t.TempDir(), "out") + if err := Unpack(&buf, dst); err != nil { + t.Fatalf("Unpack: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(dst, "a.txt")); string(got) != "alpha" { + t.Errorf("a.txt = %q", got) + } + if got, _ := os.ReadFile(filepath.Join(dst, "sub", "deep", "b.txt")); string(got) != "beta" { + t.Errorf("b.txt = %q", got) + } + if st, err := os.Stat(filepath.Join(dst, "empty")); err != nil || !st.IsDir() { + t.Errorf("empty dir not restored: %v", err) + } + if target, err := os.Readlink(filepath.Join(dst, "link")); err != nil || target != "a.txt" { + t.Errorf("symlink = %q, %v", target, err) + } + if st, _ := os.Stat(filepath.Join(dst, "a.txt")); st.Mode()&0o100 == 0 { + t.Errorf("executable bit lost: %v", st.Mode()) + } +} + +func TestPackMissingDir(t *testing.T) { + if err := Pack(filepath.Join(t.TempDir(), "nope"), io.Discard); err == nil { + t.Fatal("Pack of a missing dir succeeded") + } +} + +func tarStream(t *testing.T, build func(tw *tar.Writer)) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + build(tw) + // A deliberately truncated entry (header without body) makes Close + // complain; the header is already on the wire, which is what the + // test wants. + _ = tw.Close() + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return &buf +} + +func TestUnpackRejectsBadStreams(t *testing.T) { + dst := t.TempDir() + if err := Unpack(strings.NewReader("not gzip"), dst); err == nil || !strings.Contains(err.Error(), "gzip") { + t.Errorf("non-gzip accepted: %v", err) + } + escape := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "../evil", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(escape, dst); err == nil || !strings.Contains(err.Error(), "escapes") { + t.Errorf("escaping entry accepted: %v", err) + } + abs := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "/etc/passwd", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(abs, dst); err == nil || !strings.Contains(err.Error(), "escapes") { + t.Errorf("absolute entry accepted: %v", err) + } + fifo := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "pipe", Typeflag: tar.TypeFifo, Mode: 0o644}) + }) + if err := Unpack(fifo, dst); err == nil || !strings.Contains(err.Error(), "unsupported type") { + t.Errorf("fifo accepted: %v", err) + } + huge := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "big", Typeflag: tar.TypeReg, Size: maxTarEntry + 1, Mode: 0o644}) + }) + if err := Unpack(huge, dst); err == nil || !strings.Contains(err.Error(), "too large") { + t.Errorf("oversized entry accepted: %v", err) + } + // Truncated tar body: gzip is fine, tar is not. + var truncated bytes.Buffer + gz := gzip.NewWriter(&truncated) + _, _ = gz.Write([]byte("garbage that is not a tar header at all")) + _ = gz.Close() + if err := Unpack(&truncated, dst); err == nil || !strings.Contains(err.Error(), "tar") { + t.Errorf("garbage tar accepted: %v", err) + } + // Destination that cannot be created. + blocker := filepath.Join(t.TempDir(), "file") + writeFile(t, blocker, "x") + empty := tarStream(t, func(*tar.Writer) {}) + if err := Unpack(empty, filepath.Join(blocker, "sub")); err == nil { + t.Errorf("unpack under a regular file succeeded") + } +} + +func TestUnpackSymlinkReplacesExistingAndNestedEntries(t *testing.T) { + dst := t.TempDir() + writeFile(t, filepath.Join(dst, "link"), "a file where the link goes") + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "target", Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "d/e/link2", Typeflag: tar.TypeSymlink, Linkname: "../x", Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "d/e/f.txt", Typeflag: tar.TypeReg, Size: 2, Mode: 0o600}) + _, _ = tw.Write([]byte("hi")) + }) + if err := Unpack(stream, dst); err != nil { + t.Fatalf("Unpack: %v", err) + } + if target, err := os.Readlink(filepath.Join(dst, "link")); err != nil || target != "target" { + t.Errorf("link = %q, %v", target, err) + } + if got, _ := os.ReadFile(filepath.Join(dst, "d", "e", "f.txt")); string(got) != "hi" { + t.Errorf("f.txt = %q", got) + } +} + +func TestUnpackEntryErrorsWhenParentIsAFile(t *testing.T) { + dst := t.TempDir() + writeFile(t, filepath.Join(dst, "blocker"), "x") + for _, hdr := range []*tar.Header{ + {Name: "blocker/sub/file", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}, + {Name: "blocker/sub/link", Typeflag: tar.TypeSymlink, Linkname: "x", Mode: 0o777}, + {Name: "blocker/sub/", Typeflag: tar.TypeDir, Mode: 0o755}, + } { + h := hdr + stream := tarStream(t, func(tw *tar.Writer) { _ = tw.WriteHeader(h) }) + if err := Unpack(stream, dst); err == nil { + t.Errorf("entry %q under a regular file succeeded", h.Name) + } + } + // A regular file whose target is an existing directory cannot be opened. + if err := os.Mkdir(filepath.Join(dst, "dir"), 0o755); err != nil { + t.Fatal(err) + } + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "dir", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(stream, dst); err == nil { + t.Errorf("regular file over a directory succeeded") + } +} + +func TestRunReportsExitCodesAndErrors(t *testing.T) { + ws := t.TempDir() + ok := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo out; echo err >&2; pwd"}}, ws) + if ok.ExitCode != 0 || ok.Error != "" || !strings.Contains(ok.Stdout, "out") || !strings.Contains(ok.Stderr, "err") { + t.Errorf("ok run = %+v", ok) + } + if evalWS, _ := filepath.EvalSymlinks(ws); !strings.Contains(ok.Stdout, filepath.Base(evalWS)) { + t.Errorf("command did not run in the workspace: %q", ok.Stdout) + } + + fail := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "exit 7"}}, ws) + if fail.ExitCode != 7 || fail.Error != "" { + t.Errorf("exit 7 run = %+v", fail) + } + + missing := Run(context.Background(), RunRequest{Command: []string{"/definitely/not/a/binary"}}, ws) + if missing.ExitCode != ExitCodeUnknown || missing.Error == "" { + t.Errorf("missing binary run = %+v", missing) + } + + env := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo $FOO:$BAR; env | grep -c ^HOME= || true"}, Env: map[string]string{"FOO": "1", "BAR": "2"}}, ws) + if !strings.HasPrefix(env.Stdout, "1:2\n0") { + t.Errorf("env not replaced wholesale: %q", env.Stdout) + } + withPath := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo $PATH"}, Env: map[string]string{"PATH": "/custom/bin"}}, ws) + if strings.TrimSpace(withPath.Stdout) != "/custom/bin" { + t.Errorf("explicit PATH not honoured: %q", withPath.Stdout) + } + + other := t.TempDir() + wd := Run(context.Background(), RunRequest{Command: []string{"pwd"}, WorkDir: other}, ws) + if evalOther, _ := filepath.EvalSymlinks(other); !strings.Contains(wd.Stdout, filepath.Base(evalOther)) { + t.Errorf("workdir not honoured: %q", wd.Stdout) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cancelled := Run(ctx, RunRequest{Command: []string{"sh", "-c", "exit 0"}}, ws) + if cancelled.Error == "" { + t.Errorf("cancelled run reported no error: %+v", cancelled) + } +} + +func TestCapString(t *testing.T) { + if got := capString(bytes.Repeat([]byte("x"), MaxOutput+10)); len(got) != MaxOutput { + t.Errorf("len = %d", len(got)) + } +} + +func newBridgeServer(t *testing.T) (*httptest.Server, string) { + t.Helper() + ws := filepath.Join(t.TempDir(), "ws") + srv := httptest.NewServer(&Handler{Workspace: ws}) + t.Cleanup(srv.Close) + return srv, ws +} + +func TestHandlerRoutes(t *testing.T) { + srv, ws := newBridgeServer(t) + + resp, err := http.Get(srv.URL + PathHealth) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("healthz: %v %v", err, resp) + } + resp.Body.Close() + + src := t.TempDir() + writeFile(t, filepath.Join(src, "hello.txt"), "hi") + var buf bytes.Buffer + if err := Pack(src, &buf); err != nil { + t.Fatal(err) + } + req, _ := http.NewRequest(http.MethodPut, srv.URL+PathWorkspace, &buf) + resp, err = http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != http.StatusNoContent { + t.Fatalf("put workspace: %v %v", err, resp) + } + resp.Body.Close() + if got, _ := os.ReadFile(filepath.Join(ws, "hello.txt")); string(got) != "hi" { + t.Errorf("workspace not unpacked: %q", got) + } + + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "echo changed > hello.txt; exit 3"}}) + resp, err = http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("run: %v %v", err, resp) + } + var out RunResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + resp.Body.Close() + if out.ExitCode != 3 { + t.Errorf("run response = %+v", out) + } + + resp, err = http.Get(srv.URL + PathWorkspace) + if err != nil || resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != contentTypeTar { + t.Fatalf("get workspace: %v %v", err, resp) + } + dst := t.TempDir() + if err := Unpack(resp.Body, dst); err != nil { + t.Fatal(err) + } + resp.Body.Close() + if got, _ := os.ReadFile(filepath.Join(dst, "hello.txt")); strings.TrimSpace(string(got)) != "changed" { + t.Errorf("pulled workspace = %q", got) + } +} + +func TestHandlerErrors(t *testing.T) { + srv, ws := newBridgeServer(t) + status := func(method, path string, body io.Reader) int { + req, _ := http.NewRequest(method, srv.URL+path, body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + return resp.StatusCode + } + if got := status(http.MethodGet, "/nope", nil); got != http.StatusNotFound { + t.Errorf("unknown route = %d", got) + } + if got := status(http.MethodDelete, PathWorkspace, nil); got != http.StatusNotFound { + t.Errorf("unknown method = %d", got) + } + if got := status(http.MethodPut, PathWorkspace, strings.NewReader("not a tarball")); got != http.StatusBadRequest { + t.Errorf("bad tarball = %d", got) + } + if got := status(http.MethodPost, PathRun, strings.NewReader("{")); got != http.StatusBadRequest { + t.Errorf("bad json = %d", got) + } + if got := status(http.MethodPost, PathRun, strings.NewReader(`{"command":[]}`)); got != http.StatusBadRequest { + t.Errorf("empty command = %d", got) + } + // Workspace missing: GET cannot pack it. + if got := status(http.MethodGet, PathWorkspace, nil); got != http.StatusInternalServerError { + t.Errorf("get of missing workspace = %d", got) + } + // Workspace parent is a file: PUT cannot recreate it. + _ = ws + blocked := httptest.NewServer(&Handler{Workspace: filepath.Join(ws, "..", "ws-file", "sub")}) + defer blocked.Close() + writeFile(t, filepath.Join(ws, "..", "ws-file"), "x") + empty := tarStream(t, func(*tar.Writer) {}) + req, _ := http.NewRequest(http.MethodPut, blocked.URL+PathWorkspace, empty) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("put under a file = %d, want 500 from clearing the workspace", resp.StatusCode) + } +} + +func TestHandlerDefaultsAndRunTimeout(t *testing.T) { + h := &Handler{} + if h.workspace() != DefaultWorkspace { + t.Errorf("default workspace = %q", h.workspace()) + } + // RunTimeout already expired: the command is killed and the response + // says why, without the test waiting on it. + srv := httptest.NewServer(&Handler{Workspace: t.TempDir(), RunTimeout: 1}) + defer srv.Close() + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "sleep 30"}}) + resp, err := http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out RunResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + if out.Error == "" { + t.Errorf("timed-out run reported no error: %+v", out) + } +} + +func TestServeStopsOnContextAndListenerClose(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- Serve(ctx, ln, &Handler{Workspace: t.TempDir()}) }() + resp, err := http.Get("http://" + ln.Addr().String() + PathHealth) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over Serve: %v %v", err, resp) + } + resp.Body.Close() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Errorf("Serve returned %v, want context.Canceled", err) + } + + ln2, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done2 := make(chan error, 1) + go func() { done2 <- Serve(context.Background(), ln2, &Handler{}) }() + // Closing the listener makes Serve return the listener error (not + // ErrServerClosed, which only Shutdown produces). + _ = ln2.Close() + if err := <-done2; err == nil { + t.Errorf("Serve returned nil after the listener closed") + } +} diff --git a/src/pkg/config/agent_sandbox_substrate_test.go b/src/pkg/config/agent_sandbox_substrate_test.go new file mode 100644 index 0000000000..3490aa1f61 --- /dev/null +++ b/src/pkg/config/agent_sandbox_substrate_test.go @@ -0,0 +1,115 @@ +package config + +import ( + "strings" + "testing" + + cruxconfig "github.com/hivecommons/hive/crux/config" + "gopkg.in/yaml.v3" +) + +// sandbox.runtime: substrate (ADR-0026 phase 2) — parse, merge semantics and +// the boot warnings that keep a misconfigured runtime from failing silently. + +func TestSandboxSubstrateParsesAndMerges(t *testing.T) { + src := ` +agent_sandbox: + enabled: true + image: img + runtime: substrate + substrate: + endpoint: api.ate-system.svc:443 + atespace: hive + token_env: ATE_TOKEN + image: ghcr.io/hivecommons/sandbox@sha256:0000000000000000000000000000000000000000000000000000000000000000 + allow_hosts: [litellm.example.com] + timeout_s: 600 + snapshot_on_idle: true +agents: + deps: + sandbox: + enabled: true + substrate: + allow_hosts: [hub.example.org] + worker_selector: { class: gvisor } +` + var cfg Config + if err := yaml.Unmarshal([]byte(src), &cfg); err != nil { + t.Fatal(err) + } + a := cfg.Agents["deps"] + if got := a.SandboxRuntime(cfg.AgentSandbox); got != SandboxRuntimeSubstrate { + t.Fatalf("runtime = %q", got) + } + eff := a.SandboxSubstrate(cfg.AgentSandbox) + if eff.Endpoint != "api.ate-system.svc:443" || eff.Atespace != "hive" || eff.TokenEnv != "ATE_TOKEN" || eff.TimeoutS != 600 || !eff.SnapshotOnIdle || !strings.Contains(eff.Image, "@sha256:") { + t.Errorf("cluster facts must fall back to the global block: %+v", eff) + } + if len(eff.AllowHosts) != 1 || eff.AllowHosts[0] != "hub.example.org" { + t.Errorf("per-agent allow_hosts must replace the global list: %v", eff.AllowHosts) + } + if eff.WorkerSelector["class"] != "gvisor" { + t.Errorf("worker_selector = %v", eff.WorkerSelector) + } + other := AgentConfig{} + if got := other.SandboxSubstrate(cfg.AgentSandbox); len(got.AllowHosts) != 1 || got.AllowHosts[0] != "litellm.example.com" { + t.Errorf("agent without a block gets the global one: %+v", got) + } +} + +func TestSandboxSubstrateGateWarnings(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]AgentConfig{ + "deps": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + }, + }, + } + cruxconfig.SetSubstratePreflight(func([]SandboxSubstrateConfig) []string { return nil }) + defer cruxconfig.SetSubstratePreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "digest-pinned") { + t.Errorf("warnings = %q, want exactly one about the digest-pinned image", got) + } + cfg.AgentSandbox.Substrate = &SandboxSubstrateConfig{Image: "ghcr.io/x/y@sha256:" + strings.Repeat("a", 64)} + if got := AgentSandboxGateWarnings(cfg); len(got) != 0 { + t.Errorf("fully configured substrate runtime still warns: %q", got) + } +} + +func TestAgentSandboxGateWarningsIncludeSubstratePreflight(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{ + Enabled: true, + Image: "img", + Substrate: &SandboxSubstrateConfig{Image: "ghcr.io/x/y@sha256:" + strings.Repeat("a", 64)}, + }, + Agents: map[string]AgentConfig{ + "a": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + "b": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + }, + }, + } + var calls, gotBlocks int + cruxconfig.SetSubstratePreflight(func(b []SandboxSubstrateConfig) []string { + calls++ + gotBlocks = len(b) + return []string{"substrate control API api.ate-system.svc:443 is unreachable"} + }) + defer cruxconfig.SetSubstratePreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") || calls != 1 || gotBlocks != 2 { + t.Errorf("warnings = %q calls=%d blocks=%d, want the preflight warning from one call over both agents", got, calls, gotBlocks) + } + + cfg.Agents = map[string]AgentConfig{"p": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimePodman}}} + calls = 0 + AgentSandboxGateWarnings(cfg) + if calls != 0 { + t.Errorf("preflight ran with no substrate agent") + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 32da9850e6..92917c4108 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1592,8 +1592,9 @@ func AgentSandboxGateWarnings(cfg *Config) []string { return nil } - var optedIn, noImage, noClaim, noGateway, badRuntime []string + var optedIn, noImage, noClaim, noGateway, noDigest, badRuntime []string var openshellBlocks []SandboxOpenShellConfig + var substrateBlocks []SandboxSubstrateConfig for name, a := range cfg.Agents { if !a.SandboxEnabled(cfg.AgentSandbox) { continue @@ -1618,12 +1619,21 @@ func AgentSandboxGateWarnings(cfg *Config) []string { if strings.TrimSpace(os.GatewayURL) != "" { openshellBlocks = append(openshellBlocks, os) } + case rt == SandboxRuntimeSubstrate: + sb := a.SandboxSubstrate(cfg.AgentSandbox) + // Substrate refuses tag-only images; the agent image is usually a + // tag, so the substrate block must carry the digest-pinned one. + if !strings.Contains(sb.Image, "@sha256:") { + noDigest = append(noDigest, name) + } + substrateBlocks = append(substrateBlocks, sb) } } sort.Strings(optedIn) sort.Strings(noImage) sort.Strings(noClaim) sort.Strings(noGateway) + sort.Strings(noDigest) sort.Strings(badRuntime) var out []string @@ -1658,6 +1668,12 @@ func AgentSandboxGateWarnings(cfg *Config) []string { "set agent_sandbox.openshell.gateway_url (or the per-agent sandbox.openshell.gateway_url) to the OpenShell gateway's gRPC address (ADR-0026)", strings.Join(noGateway, ", "))) } + if len(noDigest) > 0 { + out = append(out, fmt.Sprintf( + "agent(s) %s use the substrate sandbox runtime but resolve no digest-pinned substrate.image; Substrate refuses tag-only images and every kick will fail — "+ + "set agent_sandbox.substrate.image (or the per-agent sandbox.substrate.image) to the sandbox image's …@sha256:… reference (ADR-0026 phase 2)", + strings.Join(noDigest, ", "))) + } if len(optedIn) < len(cfg.Agents) { out = append(out, fmt.Sprintf( "agent_sandbox.enabled is true but only %d of %d agent(s) are opted in (%s); the rest still run unconfined on the tmux path (#4918)", @@ -1666,6 +1682,7 @@ func AgentSandboxGateWarnings(cfg *Config) []string { // Last, because it may reach the network (bounded and cached): the static // warnings above must not wait on a gateway. out = append(out, cruxconfig.OpenShellPreflightWarnings(openshellBlocks)...) + out = append(out, cruxconfig.SubstratePreflightWarnings(substrateBlocks)...) return out } diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index 077db8fb3a..6e8998cc56 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -38,6 +38,7 @@ type ( SandboxJobToleration = cruxconfig.SandboxJobToleration SandboxJobVolume = cruxconfig.SandboxJobVolume SandboxOpenShellConfig = cruxconfig.SandboxOpenShellConfig + SandboxSubstrateConfig = cruxconfig.SandboxSubstrateConfig StatsDisplayEntry = cruxconfig.StatsDisplayEntry ToolApprovalConfig = cruxconfig.ToolApprovalConfig ToolApprovalRule = cruxconfig.ToolApprovalRule @@ -143,6 +144,7 @@ const ( RotationHeadroomSourceCCLeft = cruxconfig.RotationHeadroomSourceCCLeft SandboxRuntimeJob = cruxconfig.SandboxRuntimeJob SandboxRuntimeOpenShell = cruxconfig.SandboxRuntimeOpenShell + SandboxRuntimeSubstrate = cruxconfig.SandboxRuntimeSubstrate SandboxRuntimePodman = cruxconfig.SandboxRuntimePodman SelfMergeMinACMMLevel = cruxconfig.SelfMergeMinACMMLevel WritableBobAPIKeyFile = cruxconfig.WritableBobAPIKeyFile diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 718fbbf15f..eeee612fb8 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -29,6 +29,25 @@ agent_sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell runtime; agent_sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) +agent_sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agent_sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) agent_sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..acmm_levels[] # TODO #11854 @@ -147,6 +166,25 @@ agents..sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell r agents..sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) +agents..sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) From 788020114d99436c39d70bf43f43591d0de915fa Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 14:45:36 -0700 Subject: [PATCH 006/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20ADR-0026=20phase?= =?UTF-8?q?=202=20=E2=80=94=20Agent=20Substrate=20runtime=20operator=20and?= =?UTF-8?q?=20deploy=20notes=20(#17745)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: ADR-0026 phase 2 — Agent Substrate runtime operator and deploy notes Amend ADR-0026 with the Substrate implementation notes (contract mapping, deviations from the reference model, cluster prerequisites and which hive clusters qualify, when to choose it over OpenShell); add the operator section to sandbox-isolation.md, the integrations table row, deploy/README pointer and a new deploy/substrate.md with install and verification steps. Part of #17695 Signed-off-by: Andrew Anderson * changelog: fragment for #17745 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🌱 changelog: make the substrate docs fragment an entry bullet Signed-off-by: Andrew Anderson * 🌱 changelog: drop the auto-generated duplicate fragment Signed-off-by: Andrew Anderson * 📖 docs: fix the ADR-0002 link in the Substrate section Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-substrate-runtime-docs.md | 1 + src/deploy/README.md | 1 + src/deploy/substrate.md | 118 ++++++++++++++++++ .../adr/0026-sandbox-runtime-selection.md | 76 ++++++++++- src/docs/integrations.md | 1 + src/docs/sandbox-isolation.md | 45 ++++++- 6 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 changelog.d/changed-substrate-runtime-docs.md create mode 100644 src/deploy/substrate.md diff --git a/changelog.d/changed-substrate-runtime-docs.md b/changelog.d/changed-substrate-runtime-docs.md new file mode 100644 index 0000000000..fadfda564d --- /dev/null +++ b/changelog.d/changed-substrate-runtime-docs.md @@ -0,0 +1 @@ +- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. diff --git a/src/deploy/README.md b/src/deploy/README.md index 6b3fb54d45..db3234b79d 100644 --- a/src/deploy/README.md +++ b/src/deploy/README.md @@ -20,6 +20,7 @@ Use it indirectly through the dashboard terminal link. If a terminal pane says i - `k8s/` — the Kustomize base (`kustomization.yaml`) with namespace, deployment, service, PVC, Secret, ConfigMap and RBAC manifests, plus the custom error backend (`error-pages.yaml`), the hub backup CronJob (`backup-cronjob.yaml`) and the node-prep DaemonSet (`node-prep/`), which are applied separately. The OpenShift Route lives in `kustomize/overlays/openshift/`. - `inference/` — sample in-cluster OpenAI-compatible inference deployment and RBAC. - `openshell.md` — cluster-side notes for the opt-in `sandbox.runtime: openshell` (ADR-0026): prerequisites, the upstream Helm install, the gateway settings and PVC labels hive's runtime depends on, and how to verify. No manifests are vendored. +- `substrate.md` — cluster-side notes for the opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2): the Agent Substrate operator install, WorkerPool and SandboxConfig, the pod-certificate feature gates, the router client bundle hive needs, and how to verify. No manifests are vendored. - `docker-compose.architect.yaml`, `hive-quickstart.yaml`, `hive-level*.yaml`, `architect-only.yaml`, `hive.yaml` — example deployment/configuration manifests. - `blue-green-deploy.sh`, `bootstrap-lxc.sh`, `create-lxc.sh` — operational scripts for **Docker Compose** deployments (LXC bootstrap and a blue-green Compose upgrade). Not "non-Kubernetes" generally: none of the three shells out to `podman` or reads `HIVE_DEPLOY_RUNTIME`, so none of them applies to the Quadlet stack in `quadlet/` below. See [Deployment helper scripts](../docs/deployment-scripts.md) for each script's scope and where a Podman operator should go instead. - `quadlet/` — Podman Quadlet units for the standalone deployment (ADR-0017). `hive.container` and `hive-data.volume` start the Hive service under `systemd`, rootful or rootless; `hive.network` and `hive-gateway.container` put the authenticating gateway in front of it with the same published-port split as the Compose stack — 3001 published, 7681 never. See [the operator guide](../docs/podman-standalone-quadlet.md). Docker Compose remains the default runtime and is unaffected. diff --git a/src/deploy/substrate.md b/src/deploy/substrate.md new file mode 100644 index 0000000000..941d32d2c3 --- /dev/null +++ b/src/deploy/substrate.md @@ -0,0 +1,118 @@ +# Installing Agent Substrate for `sandbox.runtime: substrate` + +Cluster-side notes for the opt-in Substrate sandbox runtime +([ADR-0026 phase 2](../docs/adr/0026-sandbox-runtime-selection.md#implementation-notes-phase-2-agent-substrate), +epic [#17695](https://github.com/hivecommons/hive/issues/17695); operator view +in [sandbox-isolation.md](../docs/sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor)). +Hive installs nothing: Substrate is a cluster-admin install, once per cluster, +and hive only talks to its Control API and router. Upstream is the source of +truth for the install — [agent-substrate/substrate → docs/guides/operator-install.md](https://github.com/agent-substrate/substrate/blob/main/docs/guides/operator-install.md), +the [API guide](https://github.com/agent-substrate/substrate/blob/main/docs/reference/api-guide.md) +and [sandbox classes](https://github.com/agent-substrate/substrate/blob/main/docs/reference/sandbox-classes.md) — +and no manifests are vendored here; pin the release you install and record it +alongside the hive config. + +## Prerequisites to check before installing + +| Requirement | Why | How to check | +|---|---|---| +| Kubernetes with `ClusterTrustBundle`, `ClusterTrustBundleProjection` and `PodCertificateRequest` feature gates and `certificates.k8s.io/v1beta1` enabled | actor and client identities are pod certificates (SPIFFE); the podcert controller does not start without them. Still off by default as of Kubernetes 1.36 | `kubectl get --raw /metrics \| grep 'kubernetes_feature_enabled{name="PodCertificateRequest"'` and `kubectl api-resources --api-group=certificates.k8s.io`. Managed control planes (OKE, LKE) need the provider to enable them; OpenShift needs a `FeatureGate` CR | +| Standard Linux nodes with **cgroup v2** for the `gvisor` class | `ateworker-gvisor` runs `runsc` inside the worker pod; there is no kernel floor beyond ≥ 5.8 | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`; `stat -fc %T /sys/fs/cgroup` on a node → `cgroup2fs` | +| `/dev/kvm` on nodes for the `microvm` class | Kata on Cloud Hypervisor guests; `atelet` advertises the `ate.dev/kvm` device only where it exists | `kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.allocatable.ate\.dev/kvm}{"\n"}{end}'` after install | +| Nodes trusted to run worker pods with `SYS_ADMIN` and `NET_ADMIN` | workers are `privileged: false` but keep those two for mounts and networking | on OpenShift, an SCC granting them to the `ate-system` worker ServiceAccount | +| A CNI that **enforces** egress `NetworkPolicy` | Substrate's `atenet` dataplane carries actor egress; the worker pod itself must not be a side door | create a deny-all policy in a scratch namespace and confirm a connection is refused | +| PostgreSQL and S3-compatible object storage | Control API state; actor snapshots (`snapshot_on_idle`) | `ate-setup` deploys bundled ones for a lab; use managed services in production | +| Cluster-admin for the install | CRDs (`workerpools`, `sandboxconfigs`, `egresspolicies`, …), ValidatingAdmissionPolicies, RBAC, PKI | — | + +### Which hive clusters qualify (survey 2026-11-11) + +| cluster | kernel / Kubernetes | `gvisor` | `microvm` | +|---|---|---|---| +| hive-oke | OL9, UEK8 6.12, k8s 1.34.10 | yes, once the podcert feature gates are enabled on the OKE control plane | needs KVM-capable shapes (bare metal or nested virtualisation) | +| lke648397, hive-ci | Debian 13, 6.12 | yes; confirm the feature gates — LKE exposes no control-plane flags | KVM to confirm per plan | +| vllm-d, a-ks-wec2 | RHCOS 9.x, 5.14 | possible: SCC for the worker capabilities + `FeatureGate` CR (TechPreviewNoUpgrade) | only on KVM-capable nodes | + +## Install + +Build or download `ate-setup` from a pinned release of +`github.com/agent-substrate/substrate`, write the install document, and deploy: + +```yaml +# substrate-install.yaml +apiVersion: install.ate.dev/v1alpha1 +kind: SubstrateInstall +namespace: ate-system +atenet: + dataplane: envoy +images: + repo: /substrate + tag: +``` + +```shell +ate-setup deploy ate-system --config substrate-install.yaml \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns//sa/ +``` + +`--ingress-allowed-spiffe-ids` must name hive's own pod identity: that is what +lets hive open `CONNECT` tunnels through `atenet-router` to the bridge in each +actor. Unknown keys in the document are rejected, not ignored. + +Then create the pieces hive's runtime depends on, in Substrate terms: + +- **An atespace** for hive (`sandbox.substrate.atespace`, default `default`), + and a principal for hive's token (`token_env` / `token_file`) that may + create and delete `ActorTemplate`, `Actor` and `EgressPolicy` and list + workers in it. `kubectl ate` shows the exact RBAC verbs for the release. +- **A `WorkerPool`** with `spec.sandboxClasses[].name: gvisor` (or `microvm`) + resolving the default `SandboxConfig` (`gvisor-default`), labelled so that + `sandbox.substrate.worker_selector` matches it; hive only uses pools whose + workers report `ACTIVE` and advertise a sandbox class. Taint the node pool + `ate.dev/sandboxClass=:NoSchedule` to reserve it; worker pods + tolerate it. +- **Projected files in the hive pod**: a ServiceAccount token with audience + `api.ate-system.svc` (→ `token_file`), the Control API CA via + `clusterTrustBundle` with signer `servicedns.podcert.ate.dev/identity` + (→ `ca_file`), and a `podCertificate` with signer + `podidentity.podcert.ate.dev/identity` (→ `router_client_bundle`) plus the + router CA (→ `router_ca_file`). The authentication guide upstream has the + volume stanza. +- **A digest-pinned sandbox image** containing `crux` (for the + `substrate-bridge` entrypoint) and the agent CLI, set as + `sandbox.substrate.image`; tags are refused at boot and at kick. Publishing + it is tracked in [#17697](https://github.com/hivecommons/hive/issues/17697). +- **Egress**: hive writes its own `EgressPolicy` per actor (`github.com`, + `api.github.com`, `allow_hosts`). Do **not** also configure Substrate's + credential injection for GitHub — hive's MITM proxy brokers credentials and + the two would conflict. + +## Verify + +```shell +kubectl -n ate-system get deploy,ds # ateapi, atenet-router, podcert controller, atelet ready +kubectl get crd | grep ate.dev # workerpools, sandboxconfigs, egresspolicies, … +kubectl get workerpool -A # at least one pool with ACTIVE workers +kubectl get sandboxconfig gvisor-default -o yaml | grep defaultVersion +kubectl get nodes -o custom-columns=NAME:.metadata.name,KVM:.status.allocatable.ate\.dev/kvm +kubectl ate create actor smoke -a --template && kubectl ate delete actor smoke -a +``` + +From the hive side, start with `runtime: substrate` and watch the boot log: the +preflight warns when the Control API is unreachable, has no matching `ACTIVE` +worker, or its workers report no sandbox class. Then run one kick and confirm +in the router log that the actor's SPIFFE id made exactly the allowed +connections and that a connect to any other host was refused — that denial is +the point of the runtime. Latency and image numbers go back into this page +from the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); +the fuller prover (CRDs, operator, feature gates, KVM) is +[#17698](https://github.com/hivecommons/hive/issues/17698). + +## Upgrades + +Substrate is pre-1.0; follow its version-skew policy (N ↔ N+1 only) and pin +the release and the sandbox image by digest. A `SandboxConfig` upgrade adds a +`runsc` version and keeps the previous one so suspended actors still restore. +Hive's generated client (`crux/sandbox/substrateapi`) tracks the fields it +uses; bump it with `generate.sh` against the matching upstream tag when the +Control API changes. diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index 3a6e062fde..153111ef6f 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -1,6 +1,6 @@ # ADR-0026: Sandbox runtime selection -Status: Accepted (phase 1 implemented, runtime opt-in) — #17642, hivecommons/hive#17646 +Status: Accepted (phase 1 OpenShell implemented, runtime opt-in — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) ## Context @@ -218,6 +218,74 @@ runs on, read on 2026-11-10: LKE (`lke648397`, `hive-ci`) run 6.12 on Debian 13 with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters (5.14) do not. +## Implementation notes (phase 2: Agent Substrate) + +Phase 2 (#17695, hivecommons/hive#17741) adds a **fourth** runtime, +`sandbox.runtime: substrate`, behind the same `sandbox.Launcher` seam. It is +[Agent Substrate](https://github.com/agent-substrate/substrate) (CNCF Sandbox, +donated by Google, Apache-2.0): a Kubernetes-hosted *actor* runtime where an +`ActorTemplate` (image, command, env, limits, volumes) is instantiated as an +`Actor` through a gRPC Control API (`ate.control.v1`), scheduled on a +`WorkerPool` whose pods run the workload under gVisor (`runsc`, class +`gvisor`) or in a KVM microVM (Kata on Cloud Hypervisor, class `microvm`). All +egress leaves through `atenet-router` under an `EgressPolicy`, and every actor +holds a SPIFFE identity issued through Kubernetes pod certificates. It was not +in the 2026-10-25 comparison; the write-up that led here is the Substrate +vs. crux research note, and the operator view is in +[sandbox-isolation.md](../sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor) +and [deploy/substrate.md](../../deploy/substrate.md). Like OpenShell it is +opt-in and inert unless selected. + +### How it maps onto the contract + +| Contract item | Substrate | What phase 2 does | +|---|---|---| +| Kernel-enforced confinement | gVisor user-space kernel or a microVM; worker pods are `privileged: false` but keep `SYS_ADMIN`/`NET_ADMIN` | selects the class through `worker_selector`; no change to hive | +| Default-deny egress | `atenet-egress` `EgressPolicy` per actor; rules are TLS-passthrough or terminate per host:port, no per-method read-only mode | `SubstrateEgressPolicy`: `github.com` and `api.github.com` on 443 plus `sandbox.substrate.allow_hosts`, TLS passthrough; everything else (and all UDP) denied | +| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | unchanged hive brokering — the agent still gets `HTTPS_PROXY` to the hive MITM proxy (ADR-0002); Substrate's injection is **not** used, the two are complementary | +| Workspace in, commits out | `durableDir` volumes and object-storage snapshots; no exec/attach API | an in-actor HTTP **bridge** (`crux substrate-bridge`) reached through the router's mTLS `CONNECT` tunnel: push tar → `POST /run` → pull tar; the hive-side workspace stays authoritative | +| Cleanup and timeouts | `DeleteActor` / `SuspendActor` / `DeleteActorTemplate` | one template + actor per kick, both deleted on success, failure and timeout (fresh bounded context, `ExitCode -1`, 16 MiB cap, as `job`/`openshell`) | + +### Deviations from Substrate's reference model + +| Substrate assumes | Phase 2 does | +|---|---| +| long-lived actors and reusable templates | per-kick `ActorTemplate` + `Actor`, named `hive-kick--` | +| Go client `go.ate.dev/substrate` | **not imported**: it needs Go 1.27 (hive is on 1.26) and a very large module graph. A trimmed, wire-compatible proto subset is generated in-repo (`crux/sandbox/substrateapi`, field numbers pinned by tests); zero new Go modules, +578 KB on the `crux` binary | +| image by tag or digest | digest-pinned `sandbox.substrate.image` only, enforced at boot (`AgentSandboxGateWarnings`) and at kick | +| no `policy_profile` concept | passed to the actor as `HIVE_POLICY_PROFILE`; Substrate has no annotations | +| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; resume-on-next-kick is not implemented (#17697) | +| `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | + +### Cluster prerequisites and which hive clusters qualify + +Substrate's real requirement is **not** a kernel floor: the gVisor class runs +`runsc` inside the worker pod on any standard Linux node with cgroup v2; the +microVM class needs `/dev/kvm` (nested virtualisation on cloud shapes). Both +need the operator (`ate-setup deploy ate-system`: Control API, `atelet` +DaemonSet, `atenet-router`, podcert controller, PostgreSQL, snapshot object +storage) and Kubernetes with the `ClusterTrustBundle`, +`ClusterTrustBundleProjection` and `PodCertificateRequest` feature gates — still +off by default as of 1.36 — because actor identities are pod certificates. +Survey on 2026-11-11: + +| cluster | kernel / Kubernetes | `gvisor` class | `microvm` class | +|---|---|---|---| +| hive-oke | OL9, UEK8 6.12, k8s 1.34.10 | yes once the feature gates are on (OKE control plane: needs a support request or a self-managed control plane) | needs KVM-capable (bare-metal or nested-virt) shapes | +| lke648397, hive-ci | Debian 13, 6.12 | yes; LKE exposes no control-plane flags, so the podcert gates are the blocker — check with `kubectl get --raw /metrics \| grep kubernetes_feature_enabled` | KVM to confirm per plan | +| vllm-d, a-ks-wec2 | RHCOS 9.x, 5.14 | possible: the kernel is fine, but the worker's `SYS_ADMIN`/`NET_ADMIN` needs an SCC and the gates a `FeatureGate` CR (TechPreviewNoUpgrade) | only on KVM-capable nodes | + +### When to choose it over OpenShell + +- Choose **OpenShell** when the cluster already runs a ≥ 6.2 kernel with an + enforcing CNI and you want per-binary, per-method egress policy and the + shared-PVC workspace with no image changes. +- Choose **Substrate** when you want a stronger boundary than Landlock + + seccomp (gVisor or a VM), snapshot/suspend of idle agents, or SPIFFE + identities for the sandboxed workload, and you can run its operator, + PostgreSQL and object storage. Its egress policy is coarser (host:port, no + read-only API mode) and the workspace is copied, not shared. + ## Consequences - Hive gains a sandbox boundary that is checked before the agent runs, without @@ -233,8 +301,12 @@ with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters hive's policy. - Telemetry must be disabled explicitly. A preflight check should refuse a gateway that reports it enabled. +- Substrate (phase 2) adds a second operator-side stack (Control API, + atelet, atenet-router, PostgreSQL, object storage) and a copied workspace; + the hive-side credential model is untouched. Its preflight (#17698) and + prototype (#17697) follow the OpenShell ones. - If the prototype fails a gate, this record is rewritten to decide "no change" with the measured reason, and the Podman and Job runtimes remain the only ones. -Refs #2804, #6311, #13806, #14092. +Refs #2804, #6311, #13806, #14092, #17642, #17695. diff --git a/src/docs/integrations.md b/src/docs/integrations.md index f4479b2e3c..fed5bf8cad 100644 --- a/src/docs/integrations.md +++ b/src/docs/integrations.md @@ -94,6 +94,7 @@ per-backend confinement matrix. Podman logoRootless Podman agent sandboxOpt-insandbox-isolation.md Kubernetes logoKubernetes Job sandbox runtimeOpt-insandbox-isolation.md NVIDIA logoNVIDIA OpenShell sandbox runtimeIn progress (opt-in, phase 1)sandbox-isolation.md · #17642 +CNCF logoAgent Substrate sandbox runtime (CNCF Sandbox)In progress (opt-in, phase 2)sandbox-isolation.md · #17695 OpenSandboxAAIF OpenSandbox (evaluation)Planned#14092 diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 55edeb2784..507b404980 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -286,6 +286,49 @@ The gateway, workspace, token, TLS and claim fields are cluster facts and fall b **Limitations (phase 1).** Kubernetes driver only (no Podman-driver path for standalone hosts yet); no Secret references into the sandbox; `policy_profile` is advisory; kernel and claim-label prerequisites are not preflighted; latency and image size are not yet measured ([#17643](https://github.com/hivecommons/hive/issues/17643)). +## Running sandboxed kicks in a Substrate actor + +`sandbox.runtime: substrate` runs an agent's sandboxed kicks as an [Agent Substrate](https://github.com/agent-substrate/substrate) **actor** ([ADR-0026 phase 2](adr/0026-sandbox-runtime-selection.md#implementation-notes-phase-2-agent-substrate), epic [#17695](https://github.com/hivecommons/hive/issues/17695)). Substrate is a CNCF Sandbox project donated by Google: a `WorkerPool` of pods runs workloads under gVisor (`runsc`) or in a KVM microVM, every actor gets a SPIFFE identity, and all egress leaves through `atenet-router` under an `EgressPolicy`. It is the fourth runtime and, like OpenShell, opt-in; nothing changes for a hive that does not select it. + +Everything above about the kick path still holds; only step 2 changes. The launcher is `sandbox.SubstrateLauncher`, which talks to Substrate's gRPC **Control API** (`ate.control.v1`) with a trimmed, wire-compatible client generated in-repo (`crux/sandbox/substrateapi`; the upstream Go module needs a newer Go than hive builds with). + +```yaml +agent_sandbox: + runtime: substrate + substrate: + endpoint: api.ate-system.svc:443 # Control API, host:port or https:// + atespace: hive # Substrate's tenancy unit, default "default" + token_env: ATE_TOKEN # bearer token env var (or token_file for a projected SA token) + ca_file: /var/run/ate/ca.crt # Control API CA; insecure_tls: true only for a lab + router_url: https://atenet-router.ate-system.svc:8444 + router_client_bundle: /var/run/ate/credential-bundle.pem # hive's own mTLS client identity + router_ca_file: /var/run/ate/router-ca.crt + image: ghcr.io/hivecommons/crux-sandbox@sha256:… # digest-pinned; tags are refused + cpu_millicores: 2000 + memory_bytes: 4294967296 + worker_selector: {ate.dev/sandboxClass: gvisor} # pick the pool (gvisor | microvm) + snapshot_on_idle: false + allow_hosts: [models.example.com, hub.example.com:8443] + timeout_s: 1800 +agents: + reviewer: + sandbox: + substrate: + snapshot_on_idle: true # per-agent overrides merge onto the global block +``` + +**One actor per kick.** Hive creates an `ActorTemplate` (one container running the bridge command, the workspace as a `durableDir`, the operator's limits and `worker_selector`) and an `Actor` from it, both named `hive-kick--`, attaches the egress policy, waits for the actor to report `RUNNING`, and then talks to a small HTTP **bridge** inside the actor (`crux substrate-bridge`, overridable with `bridge_command`, port `bridge_port`) through the router's mTLS `CONNECT` tunnel: it pushes the workspace as a tar, runs the kick with the sanitized environment, and pulls the workspace back. The hive-side workspace remains the source of truth and feeds the push broker exactly as for Podman. Substrate has no exec API and hive shares no PVC with the worker nodes, which is why the workspace is copied rather than mounted; the copy is bounded by the kick timeout. + +**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Credential brokering is unchanged: the agent inside the actor still receives `HTTPS_PROXY` pointing at the hive MITM proxy ([ADR-0002](adr/0002-mitm-proxy-network-enforcement.md)); Substrate's own zero-credential injection is complementary and not used by hive. + +**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. + +**What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). + +**The terminal pane.** A substrate-runtime agent has no tmux session and, until [#17699](https://github.com/hivecommons/hive/issues/17699), no live pane: the ▶ terminal shows the last persisted transcript like the job viewer between kicks. + +**Limitations (phase 2).** Workspace copied, not shared; `crux substrate-bridge` entrypoint and a published sandbox image land with the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); no resume of suspended actors; `api.github.com` not read-only at the policy layer; latency and image size not yet measured. + ## Health and metrics for sandboxed agents A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. @@ -315,7 +358,7 @@ On-demand agents, paused agents and `pause`/`continuous` cadences are never over - The Job runtime is a first slice: one Job per kick, workspace shared through a PVC, no per-kick Secret minting. A clone-inside-the-Job transport for clusters without ReadWriteMany storage is future work. - Live Podman execution is covered by skip-when-absent tests; CI still needs a rootless-Podman runner lane for always-on integration coverage. - Sandboxed inference depends on an operator-provided restricted network/proxy policy. `none` is stronger but not yet usable for all model backends. -- A stronger runtime than plain containers is under evaluation: [ADR-0026](adr/0026-sandbox-runtime-selection.md) (Proposed) compares NVIDIA OpenShell and OpenSandbox against this contract. +- Stronger runtimes than plain containers ship opt-in: [ADR-0026](adr/0026-sandbox-runtime-selection.md) (Accepted) covers NVIDIA OpenShell (phase 1) and Agent Substrate (phase 2); both still need their prototype measurements and cluster preflights. ## Agent guardrails (defense in depth) From f0b35eb9e728e35792e6a4464769fa04b1471abb Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 21:59:41 +0000 Subject: [PATCH 007/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.221.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.221.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 8 ++++++++ changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md | 1 - changelog.d/fixed-discovery-binary-stderr.md | 1 - changelog.d/fixed-gitconfig-identity-warning-once.md | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md delete mode 100644 changelog.d/fixed-discovery-binary-stderr.md delete mode 100644 changelog.d/fixed-gitconfig-identity-warning-once.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f1a05a1eaa..c95a8aeea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.221.1) + +### Fixed + +- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). +- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). +- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). + ## 2026-11-11 (v5.221.0) ### Added diff --git a/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md b/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md deleted file mode 100644 index 80a3ef1606..0000000000 --- a/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md +++ /dev/null @@ -1 +0,0 @@ -- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). diff --git a/changelog.d/fixed-discovery-binary-stderr.md b/changelog.d/fixed-discovery-binary-stderr.md deleted file mode 100644 index 18be9f2d91..0000000000 --- a/changelog.d/fixed-discovery-binary-stderr.md +++ /dev/null @@ -1 +0,0 @@ -- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). diff --git a/changelog.d/fixed-gitconfig-identity-warning-once.md b/changelog.d/fixed-gitconfig-identity-warning-once.md deleted file mode 100644 index 55515cc892..0000000000 --- a/changelog.d/fixed-gitconfig-identity-warning-once.md +++ /dev/null @@ -1 +0,0 @@ -- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). From 21053187d48aced73a7b5593d32fdf6499aba53b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:18:35 +0000 Subject: [PATCH 008/223] fix: stop read-only lanes retrying hive:unauthorized label removal (#17743) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...nauthorized-label-removal-readonly-lane.md | 1 + .../automerge/automerge_unauthorized.go | 6 ++ .../automerge_unauthorized_readonly_test.go | 58 +++++++++++++++++++ src/pkg/github/integration_read_only.go | 7 +++ 4 files changed, 72 insertions(+) create mode 100644 changelog.d/fixed-unauthorized-label-removal-readonly-lane.md create mode 100644 src/pkg/github/automerge/automerge_unauthorized_readonly_test.go diff --git a/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md b/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md new file mode 100644 index 0000000000..9939b3a538 --- /dev/null +++ b/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md @@ -0,0 +1 @@ +- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). diff --git a/src/pkg/github/automerge/automerge_unauthorized.go b/src/pkg/github/automerge/automerge_unauthorized.go index a872bf1ff8..d72e68fc7d 100644 --- a/src/pkg/github/automerge/automerge_unauthorized.go +++ b/src/pkg/github/automerge/automerge_unauthorized.go @@ -165,6 +165,12 @@ func (c *Engine) clearUnauthorized(ctx context.Context, displayRepo, owner, repo if !had && !labelled { return } + if labelled && hgithub.IntegrationReadOnly(ctx) { + // The read-only lane probe refuses the DELETE; a write-capable lane + // clears the label, so retrying here would only warn every cycle. + c.debug("lane unauthorized: label removal left to a write-capable lane", "repo", displayRepo, "pr", number, "label", unauthorizedLabel, "step", step) + return + } if labelled { if _, err := c.gh.Issues.RemoveLabelForIssue(hgithub.WithRESTCaller(ctx, unauthorizedRESTCaller), owner, repo, number, url.PathEscape(unauthorizedLabel)); err != nil && !isGitHubStatus(err, http.StatusNotFound) { c.warn("lane unauthorized: could not remove label", "repo", displayRepo, "pr", number, "label", unauthorizedLabel, "error", err) diff --git a/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go b/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go new file mode 100644 index 0000000000..765c435926 --- /dev/null +++ b/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go @@ -0,0 +1,58 @@ +package automerge + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" +) + +func TestClearUnauthorizedReadOnlyLaneSkipsLabelRemoval(t *testing.T) { + for _, tc := range []struct { + name string + readOnly bool + }{ + {"read-only lane", true}, + {"write-capable lane", false}, + } { + t.Run(tc.name, func(t *testing.T) { + var deletes atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete && strings.HasSuffix(r.URL.Path, "/issues/7/labels/"+unauthorizedLabel) { + deletes.Add(1) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + c := newContributorLaneEngine(srv.URL, ContributorLanePolicy{}, nil, Options{}) + var logs bytes.Buffer + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo})) + pr := &gh.PullRequest{Labels: []*gh.Label{{Name: gh.Ptr(unauthorizedLabel)}}} + ctx := context.Background() + if tc.readOnly { + ctx = hgithub.WithIntegrationReadOnly(ctx) + } + for range 3 { + c.clearUnauthorized(ctx, "acme/widget", "acme", "widget", pr, 7, "approve-workflow-runs") + } + if tc.readOnly { + if got := deletes.Load(); got != 0 { + t.Fatalf("deletes = %d, want 0", got) + } + } else if deletes.Load() < 1 { + t.Fatal("write-capable lane did not remove the label") + } + if strings.Contains(logs.String(), "level=WARN") { + t.Fatalf("unexpected warning: %s", logs.String()) + } + }) + } +} diff --git a/src/pkg/github/integration_read_only.go b/src/pkg/github/integration_read_only.go index dace3f27ae..ab0fa0f93c 100644 --- a/src/pkg/github/integration_read_only.go +++ b/src/pkg/github/integration_read_only.go @@ -24,6 +24,13 @@ func WithIntegrationReadOnly(ctx context.Context) context.Context { return context.WithValue(ctx, integrationReadOnlyKey{}, true) } +// IntegrationReadOnly reports whether ctx carries WithIntegrationReadOnly, so +// callers can skip writes the transport would refuse anyway. +func IntegrationReadOnly(ctx context.Context) bool { + guarded, _ := ctx.Value(integrationReadOnlyKey{}).(bool) + return guarded +} + func integrationReadOnlyRequest(request *http.Request) error { if guarded, _ := request.Context().Value(integrationReadOnlyKey{}).(bool); !guarded { return nil From 82f51ef0d7147bef9b36c4f932d1df7b5a01e302 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:19:01 +0000 Subject: [PATCH 009/223] fix: reconcile and expire effect-ledger ops stuck in Unknown (#17746) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ixed-effect-ledger-unknown-op-reconcile.md | 1 + src/cmd/hive/selfreport_mutation_hooks.go | 3 +- src/crux/effects/effects.go | 7 + src/crux/effects/execute_reconciling_test.go | 29 ++++ src/docs/fleet-report.md | 2 +- src/docs/troubleshooting.md | 4 +- src/pkg/convergence/mutation/boundary.go | 51 ++++++ .../mutation/boundary_unresolved_ttl_test.go | 156 ++++++++++++++++++ src/pkg/effects/effects.go | 1 + src/pkg/github/issue_request_watcher.go | 2 + 10 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-effect-ledger-unknown-op-reconcile.md create mode 100644 src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go diff --git a/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md b/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md new file mode 100644 index 0000000000..cd12604414 --- /dev/null +++ b/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md @@ -0,0 +1 @@ +- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). diff --git a/src/cmd/hive/selfreport_mutation_hooks.go b/src/cmd/hive/selfreport_mutation_hooks.go index 48e7ac77d1..1d450607e4 100644 --- a/src/cmd/hive/selfreport_mutation_hooks.go +++ b/src/cmd/hive/selfreport_mutation_hooks.go @@ -12,7 +12,8 @@ import ( // hooks to the self-report tracker: a journaled external effect that stays // Unknown/Planned past mutation.DefaultStuckAfter while callers keep retrying // it is reported as fleetreport.ClassEffectLedgerOpStuck (key = the op's -// logical ID), and the condition clears when the op is applied or reconciled. +// logical ID), and the condition clears when the op is applied, reconciled or +// expired past its kind's unresolved TTL (Boundary.UnresolvedTTL). func installMutationSelfReportHooks(b *mutation.Boundary) { if b == nil { return diff --git a/src/crux/effects/effects.go b/src/crux/effects/effects.go index ed3491670f..6eb3c962a8 100644 --- a/src/crux/effects/effects.go +++ b/src/crux/effects/effects.go @@ -43,6 +43,12 @@ var ErrNotApplied = errors.New("external effect definitely not applied") // must be reconciled (see Reconcile) before any retry. var ErrNeedsReconciliation = errors.New("external effect needs reconciliation before retry") +// ErrUnresolvedExpired accompanies ErrNeedsReconciliation when the boundary +// refused this attempt but, finding the operation unresolved past its TTL, +// expired it to not-applied: the effect did not run, and the caller's next +// attempt (after its own duplicate check) may. +var ErrUnresolvedExpired = errors.New("unresolved external effect expired past TTL; retry authorized") + // NotApplied wraps err as a definite refusal (ErrNotApplied) while keeping // err's own message and chain, so callers still see the original error. func NotApplied(err error) error { @@ -146,6 +152,7 @@ func ExecuteReconciling(ctx context.Context, boundary Boundary, claim Claim, obs if expErr != nil { return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) } + expired = expired || errors.Is(journalErr, ErrUnresolvedExpired) if !expired { return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) } diff --git a/src/crux/effects/execute_reconciling_test.go b/src/crux/effects/execute_reconciling_test.go index 24d473bf07..558ca6b3d4 100644 --- a/src/crux/effects/execute_reconciling_test.go +++ b/src/crux/effects/execute_reconciling_test.go @@ -3,6 +3,7 @@ package effects import ( "context" "errors" + "fmt" "strings" "testing" "time" @@ -192,3 +193,31 @@ func TestLoggingBoundaryLookupAppliedForwards(t *testing.T) { t.Fatal("LoggingBoundary over a non-Lookup must not report applied") } } + +// A boundary that already expired the operation while refusing (the refusal +// carries ErrUnresolvedExpired) authorizes the retry even though its own +// ExpireUnresolved now finds nothing unresolved to expire. +func TestExecuteReconcilingBoundaryExpiredRefusalRetries(t *testing.T) { + b := &expiredOnRefusalBoundary{} + ran := 0 + res, err := ExecuteReconciling(context.Background(), b, reconcilingClaim, + func(context.Context) (ExternalState, error) { return ExternalState{}, errors.New("api down") }, + time.Minute, okEffect(&ran)) + if err != nil || ran != 1 || res.Provenance != "ran" { + t.Fatalf("got (%+v, %v, ran=%d), want the effect to run once", res, err, ran) + } +} + +type expiredOnRefusalBoundary struct{ refused bool } + +func (b *expiredOnRefusalBoundary) Execute(ctx context.Context, _ Claim, effect func(context.Context) (Result, error)) (Result, error) { + if !b.refused { + b.refused = true + return Result{}, fmt.Errorf("%w: %w", ErrNeedsReconciliation, ErrUnresolvedExpired) + } + return effect(ctx) +} + +func (b *expiredOnRefusalBoundary) ExpireUnresolved(context.Context, Claim, time.Duration) (bool, error) { + return false, nil +} diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 134fada243..be45d81494 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -137,7 +137,7 @@ title is `[selfreport] : `. | `agent_crash_loop` | the watchdog latches an agent as crash-looping and pauses it (acting mode only — observe mode would re-count every sweep); the healthy-window reset clears it. Diagnostics carry the classifier detail, failure count/cap, backoff ladder and healthy-reset window (#13470, was "agent crash loop") | high | | `ci_poll_nudge` | the harness sent a non-monitoring agent the stop-polling-CI nudge (at most one per kick); reviewer, CI-maintainer and quality lanes are exempt because reading CI status across PRs and repositories is their assigned work. Only a command line run more than once since the kick counts as polling (#14016): a sweep that reads each of several PRs' checks once does not cross the count threshold. A self-report fires only after an eligible agent ran ≥2 such repeated polls since the kick — a single nudged poll or a sweep does not trigger a report. One per 24h window; message carries polls since kick / time since kick / nudges since boot, diagnostics the scrubbed pane tail plus the repeated CI-status command lines with their counts (#14289; #13470, was "CI-poll nudge") | low | | `issue_request_stuck` | an issue/comment/label request file in the relay queue (`issue-requests/`) has failed ≥3 attempts **or** has been retrying for ≥30 min. Key is the request file name; message carries kind, attempts and age, diagnostics the last error (scrubbed). Clears when the request completes or is quarantined (`.failed`/`.bad`) | medium | -| `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, or reconciled | medium | +| `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, reconciled, or — for `issue_create` — expired to NotApplied once a refused retry finds it unresolved past 1h (the caller's next attempt re-runs its duplicate search, then creates; #17693) | medium | | `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `pkg/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | | `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. The record carries `backend` (which provider to re-authenticate), `login_hint` (the CLI's login command) and `credential_valid`; re-login in the agent's tmux pane, then Resume. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/docs/troubleshooting.md b/src/docs/troubleshooting.md index a94e34ef0b..b6ecc797a3 100644 --- a/src/docs/troubleshooting.md +++ b/src/docs/troubleshooting.md @@ -802,7 +802,9 @@ when self-reporting is enabled, as a `[selfreport]` issue. mutation journal (`/data/convergence/mutation/journal.json` in the pod). Check whether the effect exists on GitHub; the owning call site reconciles it on its next pass, or the op resolves once the request behind it is removed. A watcher retrying - a doomed request keeps the op pinned, so fix the request first. + a doomed request keeps the op pinned, so fix the request first. An + `issue_create` op still refused after 1h is expired to NotApplied by the + boundary itself and the report clears. - **`unclassified_error_repeated`** — any WARN/ERROR line repeated five times in ten minutes. The report's message is the normalised line; grep the log for it (`kubectl -n hive logs deploy/hive | grep -F ""`) to find diff --git a/src/pkg/convergence/mutation/boundary.go b/src/pkg/convergence/mutation/boundary.go index 9dd6fd9190..1411c30763 100644 --- a/src/pkg/convergence/mutation/boundary.go +++ b/src/pkg/convergence/mutation/boundary.go @@ -40,6 +40,34 @@ type Boundary struct { // leaves the unresolved state: applied, already applied, or reconciled. OnResolved func(logicalID string) StuckAfter time.Duration + // UnresolvedTTL bounds, per effect kind, how long a journaled operation + // may stay Planned/Unknown while Execute keeps refusing retries of it. + // Past it the refused call expires the operation to NotApplied (see + // ExpireUnresolved) and reports it resolved, so the stuck alert stops and + // the caller's NEXT attempt — which runs its own duplicate check first — + // may proceed. The refused call itself never runs the effect. Nil uses + // DefaultUnresolvedTTLs; a missing or non-positive entry keeps refusing + // until an explicit Reconcile. + UnresolvedTTL map[string]time.Duration +} + +// DefaultUnresolvedTTLs returns the per-kind unresolved TTLs a Boundary uses +// when UnresolvedTTL is nil. Only issue creates are listed: every issue +// create caller searches for an existing twin before it reaches Execute, so +// an Unknown create that is still refused an hour later was never found and +// a fresh attempt cannot duplicate a visible issue. Without this an advisory +// issue create left Unknown by an ambiguous response stayed refused and +// reported stuck forever (hivecommons/hive#17693). +func DefaultUnresolvedTTLs() map[string]time.Duration { + return map[string]time.Duration{effects.KindIssueCreate: effects.DefaultUnresolvedTTL} +} + +func (b *Boundary) unresolvedTTL(kind string) time.Duration { + ttls := b.UnresolvedTTL + if ttls == nil { + ttls = DefaultUnresolvedTTLs() + } + return ttls[kind] } // ExecuteJournaled preserves legacy Execute's off-mode passthrough while @@ -162,6 +190,10 @@ func (b *Boundary) Execute(ctx context.Context, claim effects.Claim, effect func return out, fmt.Errorf("%w: %v", effects.ErrDenied, execErr) } if errors.Is(execErr, ErrNeedsReconciliation) { + if b.expireRefused(ctx, claim) { + b.noteResolved(logicalID) + return out, fmt.Errorf("%w: %w: %w", effects.ErrNeedsReconciliation, execErr, effects.ErrUnresolvedExpired) + } b.noteStuck(executor, logicalID) return out, fmt.Errorf("%w: %w", effects.ErrNeedsReconciliation, execErr) } @@ -195,6 +227,25 @@ func (b *Boundary) noteStuck(executor Executor, logicalID string) { b.OnStuck(op, age) } +// expireRefused expires claim's refused operation to NotApplied once it has +// been unresolved for longer than its kind's UnresolvedTTL. It never runs the +// effect: the retry is left to the caller's next attempt. +func (b *Boundary) expireRefused(ctx context.Context, claim effects.Claim) bool { + ttl := b.unresolvedTTL(claim.Kind) + if ttl <= 0 { + return false + } + expired, err := b.ExpireUnresolved(ctx, claim, ttl) + if err != nil { + if b.Logger != nil { + b.Logger.Warn("expiring unresolved external mutation failed", + "repo", claim.Repo, "kind", claim.Kind, "target", claim.Target, "error", err) + } + return false + } + return expired +} + func (b *Boundary) noteResolved(logicalID string) { if b.OnResolved != nil && logicalID != "" { b.OnResolved(logicalID) diff --git a/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go b/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go new file mode 100644 index 0000000000..7ae2e659aa --- /dev/null +++ b/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go @@ -0,0 +1,156 @@ +package mutation + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/effects" +) + +// hivecommons/hive#17693: an issue create left Unknown by an ambiguous +// response, whose caller only ever calls Execute (no Reconcile/Expire of its +// own), was refused and reported stuck every cycle for 15h. Past the kind's +// unresolved TTL the refused call must expire the op to NotApplied, clear the +// stuck alert and — without running the effect in that call — let the +// caller's next attempt create. +func TestBoundaryExpiresRefusedUnknownIssueCreatePastTTL(t *testing.T) { + b := testBoundary(t, "enforce") + now := time.Unix(100, 0) + b.Executor.Now = func() time.Time { return now } + stuck := 0 + var resolved []string + b.OnStuck = func(Operation, time.Duration) { stuck++ } + b.OnResolved = func(id string) { resolved = append(resolved, id) } + claim := effects.Claim{Repo: "acme/widget", Kind: effects.KindIssueCreate, Target: "advisory"} + id := boundaryEffect(claim).LogicalID() + mustNotRun := func(context.Context) (effects.Result, error) { + t.Fatal("effect must not run on a refused attempt") + return effects.Result{}, nil + } + + ambiguous := errors.New("502 Bad Gateway") + if _, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, ambiguous + }); !errors.Is(err, ambiguous) { + t.Fatalf("first execute err = %v", err) + } + + // Inside the TTL: refused and reported stuck, still Unknown. + now = now.Add(DefaultStuckAfter) + _, err := b.Execute(context.Background(), claim, mustNotRun) + if !errors.Is(err, effects.ErrNeedsReconciliation) || errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("inside TTL err = %v, want a plain refusal", err) + } + if op, _ := b.Executor.Journal.Get(id); op.Status != StatusUnknown || stuck != 1 { + t.Fatalf("inside TTL: status=%s stuck=%d, want Unknown and 1", op.Status, stuck) + } + + // Past the TTL: still refused this call, but expired and resolved. + now = now.Add(effects.DefaultUnresolvedTTL) + _, err = b.Execute(context.Background(), claim, mustNotRun) + if !errors.Is(err, effects.ErrNeedsReconciliation) || !errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("past TTL err = %v, want refusal carrying ErrUnresolvedExpired", err) + } + if op, _ := b.Executor.Journal.Get(id); op.Status != StatusNotApplied { + t.Fatalf("past TTL status = %s, want NotApplied", op.Status) + } + if stuck != 1 { + t.Fatalf("stuck = %d after expiry, want no further report", stuck) + } + if len(resolved) != 1 || resolved[0] != id { + t.Fatalf("resolved = %v, want [%s]", resolved, id) + } + + // The next attempt is a fresh one and runs the effect. + ran := false + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + ran = true + return effects.Result{Provenance: "https://github.example/acme/widget/issues/9"}, nil + }) + if err != nil || !ran || res.Provenance == "" { + t.Fatalf("retry = (%+v, %v, ran=%v), want the create to run", res, err, ran) + } + if stuck != 1 { + t.Fatalf("stuck = %d after successful retry", stuck) + } +} + +// Reconciliation still wins inside the TTL: an issue found externally +// resolves the op Applied (replayed without creating), one definitively +// absent releases a retry. +func TestBoundaryReconcileResolvesUnknownIssueCreate(t *testing.T) { + for _, tc := range []struct { + name string + state effects.ExternalState + status string + runs bool + }{ + {"found", effects.ExternalState{Applied: true, Provenance: "https://github.example/acme/widget/issues/7"}, StatusApplied, false}, + {"absent", effects.ExternalState{Applied: false}, StatusNotApplied, true}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testBoundary(t, "enforce") + claim := effects.Claim{Repo: "acme/widget", Kind: effects.KindIssueCreate, Target: "finding"} + _, _ = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("connection reset") + }) + if err := b.Reconcile(context.Background(), claim, tc.state); err != nil { + t.Fatal(err) + } + if op, _ := b.Executor.Journal.Get(boundaryEffect(claim).LogicalID()); op.Status != tc.status { + t.Fatalf("status = %s, want %s", op.Status, tc.status) + } + ran := false + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + ran = true + return effects.Result{Provenance: "https://github.example/acme/widget/issues/8"}, nil + }) + if err != nil || ran != tc.runs { + t.Fatalf("after reconcile: err=%v ran=%v, want ran=%v", err, ran, tc.runs) + } + if !tc.runs && res.Provenance != tc.state.Provenance { + t.Fatalf("replay provenance = %q, want %q", res.Provenance, tc.state.Provenance) + } + }) + } +} + +// Kinds without an unresolved TTL keep refusing (and reporting stuck) until +// an explicit Reconcile, however old the entry; an explicit empty map +// disables the default for issue creates too. +func TestBoundaryUnresolvedTTLOnlyForConfiguredKinds(t *testing.T) { + for _, tc := range []struct { + name string + kind string + ttls map[string]time.Duration + }{ + {"merge default", effects.KindPullRequestMerge, nil}, + {"issue create disabled", effects.KindIssueCreate, map[string]time.Duration{}}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testBoundary(t, "enforce") + now := time.Unix(100, 0) + b.Executor.Now = func() time.Time { return now } + b.UnresolvedTTL = tc.ttls + stuck := 0 + b.OnStuck = func(Operation, time.Duration) { stuck++ } + claim := effects.Claim{Repo: "acme/widget", Kind: tc.kind, Target: "7"} + _, _ = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("timeout") + }) + now = now.Add(48 * time.Hour) + _, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + t.Fatal("effect must not run") + return effects.Result{}, nil + }) + if !errors.Is(err, effects.ErrNeedsReconciliation) || errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("err = %v, want a plain refusal", err) + } + if op, _ := b.Executor.Journal.Get(boundaryEffect(claim).LogicalID()); op.Status != StatusUnknown || stuck != 1 { + t.Fatalf("status=%s stuck=%d, want Unknown and 1", op.Status, stuck) + } + }) + } +} diff --git a/src/pkg/effects/effects.go b/src/pkg/effects/effects.go index a7066d3756..f57ef11014 100644 --- a/src/pkg/effects/effects.go +++ b/src/pkg/effects/effects.go @@ -28,6 +28,7 @@ var ( ErrDenied = cruxeffects.ErrDenied ErrNotApplied = cruxeffects.ErrNotApplied ErrNeedsReconciliation = cruxeffects.ErrNeedsReconciliation + ErrUnresolvedExpired = cruxeffects.ErrUnresolvedExpired ) type ( diff --git a/src/pkg/github/issue_request_watcher.go b/src/pkg/github/issue_request_watcher.go index ff789da3a8..51a9de7fc7 100644 --- a/src/pkg/github/issue_request_watcher.go +++ b/src/pkg/github/issue_request_watcher.go @@ -1782,6 +1782,8 @@ func (c *Client) reconcileIssueCreate(ctx context.Context, boundary effects.Boun if expErr != nil { return effects.Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) } + // The boundary may already have expired the entry while refusing. + expired = expired || errors.Is(journalErr, effects.ErrUnresolvedExpired) if !expired { return effects.Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) } From e019da8a054950416be0557e06063650e073086b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:20:04 +0000 Subject: [PATCH 010/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20guard=20nil=20con?= =?UTF-8?q?fig=20in=20stageEngineOptions=20before=20reading=20WorkflowsDir?= =?UTF-8?q?=20(#17765)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stageEngineOptions dereferenced cfg.Data.WorkflowsDir as an argument before any nil check. Because stageEngine is a package-level sync.Once, the panic only fired when a nil-config caller was first in the process, making it order-dependent. Build the workflow store only when cfg != nil. Closes #17736 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-stage-engine-nil-config.md | 1 + src/cmd/hive/stage_engine.go | 8 +++-- src/cmd/hive/stage_engine_nilcfg_test.go | 32 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-stage-engine-nil-config.md create mode 100644 src/cmd/hive/stage_engine_nilcfg_test.go diff --git a/changelog.d/fixed-stage-engine-nil-config.md b/changelog.d/fixed-stage-engine-nil-config.md new file mode 100644 index 0000000000..25490b8df1 --- /dev/null +++ b/changelog.d/fixed-stage-engine-nil-config.md @@ -0,0 +1 @@ +- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). diff --git a/src/cmd/hive/stage_engine.go b/src/cmd/hive/stage_engine.go index 6038b3da79..f856c9bfcb 100644 --- a/src/cmd/hive/stage_engine.go +++ b/src/cmd/hive/stage_engine.go @@ -39,6 +39,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op if logger == nil { logger = slog.Default() } + var wf *wfstore.Store + if cfg != nil { + wf = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + } stageEngine.opts = &stageengine.Options{ Mode: stageEngineConfiguredMode(cfg), Store: stagelabels.Shared(), @@ -46,10 +50,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op PR: stageengine.PRDeriver(), Logger: logger, Metrics: stageengine.DefaultCounters(), - WorkflowFor: stageEngineWorkflowFor(cfg, wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)), logger), + WorkflowFor: stageEngineWorkflowFor(cfg, wf, logger), } if cfg != nil { - stageEngine.opts.WorkflowProvisioned = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + stageEngine.opts.WorkflowProvisioned = wf stageEngine.opts.WorkflowLabelGC = func() bool { return cfg.Workflow.LabelGCEnabled() } } }) diff --git a/src/cmd/hive/stage_engine_nilcfg_test.go b/src/cmd/hive/stage_engine_nilcfg_test.go new file mode 100644 index 0000000000..951b091a47 --- /dev/null +++ b/src/cmd/hive/stage_engine_nilcfg_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "sync" + "testing" +) + +// A nil config must not panic while building the process-wide options +// (hivecommons/hive#17736); the sync.Once is reset so the test is the first caller. +func TestStageEngineOptions_NilConfigDoesNotPanic(t *testing.T) { + savedOpts := stageEngine.opts + stageEngine.once = sync.Once{} + stageEngine.opts = nil + t.Cleanup(func() { + stageEngine.once = sync.Once{} + stageEngine.opts = savedOpts + if savedOpts != nil { + stageEngine.once.Do(func() {}) + } + }) + + opts := stageEngineOptions(nil, nil) + if opts == nil { + t.Fatal("stageEngineOptions(nil, nil) returned nil Options") + } + if opts.WorkflowFor != nil { + t.Error("WorkflowFor should be nil without a config") + } + if opts.WorkflowProvisioned != nil { + t.Error("WorkflowProvisioned should be nil without a config") + } +} From 1c10569e98e817121d987ec3fc54a295f7c9f242 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:26:41 +0100 Subject: [PATCH 011/223] =?UTF-8?q?=F0=9F=90=9B=20Skip=20archived=20reposi?= =?UTF-8?q?tories=20during=20GFI=20curation=20(#17739)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix GFI curation for archived repositories Hive-Run: hivecommons/hive#17734 Hive-Plan: archived-repository GFI recovery Hive-Spec: good-first-issue#archived-repositories Signed-off-by: Karibhou * fix: correct changelog fragment category Signed-off-by: Karibhou --------- Signed-off-by: Karibhou Co-authored-by: Karibhou Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-gfi-archived-repositories.md | 1 + src/docs/good-first-issue.md | 5 ++ src/pkg/gfi/gfi.go | 29 +++++++++++ src/pkg/gfi/gfi_test.go | 48 +++++++++++++++++++ src/pkg/github/gfi_forge.go | 13 +++++ src/pkg/github/gfi_forge_test.go | 22 +++++++++ 6 files changed, 118 insertions(+) create mode 100644 changelog.d/fixed-gfi-archived-repositories.md diff --git a/changelog.d/fixed-gfi-archived-repositories.md b/changelog.d/fixed-gfi-archived-repositories.md new file mode 100644 index 0000000000..4d7c472e30 --- /dev/null +++ b/changelog.d/fixed-gfi-archived-repositories.md @@ -0,0 +1 @@ +- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. diff --git a/src/docs/good-first-issue.md b/src/docs/good-first-issue.md index ab7a63fdcf..cbee5f9d1f 100644 --- a/src/docs/good-first-issue.md +++ b/src/docs/good-first-issue.md @@ -121,6 +121,11 @@ wins, oldest first on ties. Issues with the `good first issue` label the hive did not apply are still given the welcome comment (once) so that their visitors get the relay pitch too. +If GitHub refuses a curation write because a repository is archived and +read-only, the sweep skips that repository until hive restarts. Remove archived +repositories from the configured repo list, or restart hive after unarchiving. +Other write failures are still reported and retried on later sweeps. + ## The welcome comment One comment per issue, found by its first-line marker ``. A diff --git a/src/pkg/gfi/gfi.go b/src/pkg/gfi/gfi.go index b1ae84d129..d05d8fcb21 100644 --- a/src/pkg/gfi/gfi.go +++ b/src/pkg/gfi/gfi.go @@ -447,6 +447,17 @@ type Forge interface { UpsertIssueMarkerComment(ctx context.Context, repo string, number int, marker, body string, create bool) (MarkerComment, error) } +// ArchivedWriteTracker remembers GitHub's explicit archived-repository write +// refusals. A nil error asks whether the repository was already recorded. +type ArchivedWriteTracker interface { + ArchivedWrite(repo string, err error) bool +} + +func archivedWrite(forge Forge, repo string, err error) bool { + tracker, ok := forge.(ArchivedWriteTracker) + return ok && tracker.ArchivedWrite(repo, err) +} + // LabelReader is the optional half of Forge the sweep uses to re-read an // issue's labels immediately before advertising it. Candidates come from // the last enumeration, which can be many minutes old; hivecommons/hive @@ -653,6 +664,10 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate labelledPerRepo := map[string]int{} for _, c := range candidates { res.Scanned++ + if archivedWrite(forge, c.Repo, nil) { + res.Skipped["archived-repo"]++ + continue + } if lowerSet(c.Labels)[Label] { labelledPerRepo[c.Repo]++ } @@ -673,6 +688,11 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate if d.AlreadyLabelled && Withdrawable(d.Skip) { if _, released := s.releasedUntil(c.Key(), p.ReselectCooldown); !released { if err := s.Withdraw(ctx, forge, c, d.Skip, opts); err != nil { + if archivedWrite(forge, c.Repo, err) { + logger.Info("gfi: repository is archived and read-only; skipping until restart", "repo", c.Repo, "error", err) + res.Skipped["archived-repo"]++ + continue + } res.Errors++ } else { res.Withdrawn = append(res.Withdrawn, c.Key()) @@ -699,6 +719,10 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate }) picked := 0 for _, sc := range list { + if archivedWrite(forge, repo, nil) { + res.Skipped["archived-repo"]++ + continue + } if picked >= p.MaxPerSweep { res.Skipped["sweep-cap"]++ logger.Debug("gfi skip", "issue", sc.c.Key(), "reason", "sweep-cap") @@ -729,6 +753,11 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate } added, err := s.apply(ctx, forge, sc.c, body) if err != nil { + if archivedWrite(forge, repo, err) { + logger.Info("gfi: repository is archived and read-only; skipping until restart", "repo", repo, "error", err) + res.Skipped["archived-repo"]++ + continue + } if errors.Is(err, errSuperseded) { res.Skipped["previously-released"]++ logger.Debug("gfi skip", "issue", sc.c.Key(), "reason", "previously-released") diff --git a/src/pkg/gfi/gfi_test.go b/src/pkg/gfi/gfi_test.go index e69e31915b..c13ed34a3c 100644 --- a/src/pkg/gfi/gfi_test.go +++ b/src/pkg/gfi/gfi_test.go @@ -327,6 +327,54 @@ func quietOpts() SweepOptions { return SweepOptions{Links: Links{Origin: "https://hive.example", HiveID: "acme-hive"}, Now: now, Logger: slog.New(slog.DiscardHandler)} } +type archivedForge struct { + *fakeForge + archived bool + attempts int +} + +var archivedErr = errors.New("archived repository") + +func (f *archivedForge) ArchivedWrite(repo string, err error) bool { + if repo != "acme/archived" { + return false + } + if errors.Is(err, archivedErr) { + f.archived = true + } + return f.archived +} + +func (f *archivedForge) UpsertIssueMarkerComment(ctx context.Context, repo string, n int, marker, body string, create bool) (MarkerComment, error) { + if repo == "acme/archived" && create { + f.attempts++ + return MarkerComment{}, archivedErr + } + return f.fakeForge.UpsertIssueMarkerComment(ctx, repo, n, marker, body, create) +} + +func TestSweepSkipsArchivedRepoAfterRefusal(t *testing.T) { + forge := &archivedForge{fakeForge: newFakeForge()} + s := NewSweeper() + a := good() + a.Repo = "acme/archived" + b := a + b.Number++ + active := good() + candidates := []Candidate{a, b, active} + first := s.Sweep(context.Background(), forge, candidates, quietOpts()) + second := s.Sweep(context.Background(), forge, candidates, quietOpts()) + if forge.attempts != 1 || first.Errors != 0 || second.Errors != 0 { + t.Fatalf("attempts=%d first=%+v second=%+v", forge.attempts, first, second) + } + if first.Skipped["archived-repo"] != 2 || second.Skipped["archived-repo"] != 2 { + t.Fatalf("archived skips: first=%+v second=%+v", first, second) + } + if len(first.Selected) != 1 || first.Selected[0] != active.Key() { + t.Fatalf("active repo not selected: %+v", first) + } +} + func TestSweepSelectsLabelsAndCommentsIdempotently(t *testing.T) { forge := newFakeForge() s := NewSweeper() diff --git a/src/pkg/github/gfi_forge.go b/src/pkg/github/gfi_forge.go index 5f1c22d387..613a0537de 100644 --- a/src/pkg/github/gfi_forge.go +++ b/src/pkg/github/gfi_forge.go @@ -17,6 +17,19 @@ func HardSuppressIssueLabels() []string { return append([]string(nil), hardSuppressIssueLabels...) } +// ArchivedWrite shares the stage sweeps' process-lifetime archive cache. +func (c *Client) ArchivedWrite(repo string, err error) bool { + if c == nil { + return false + } + owner, name := c.splitRepo(repo) + key := strings.ToLower(owner + "/" + name) + if err != nil { + return c.noteArchivedWrite(key, err) + } + return c.stageArchived.has(key) +} + // UpsertIssueMarkerComment implements gfi.Forge: it finds the first comment // on repo#number containing marker, edits it to body when body is non-empty // and differs, and — only when create is set — posts body when none exists. diff --git a/src/pkg/github/gfi_forge_test.go b/src/pkg/github/gfi_forge_test.go index a577f1482c..1a954e73a4 100644 --- a/src/pkg/github/gfi_forge_test.go +++ b/src/pkg/github/gfi_forge_test.go @@ -10,6 +10,7 @@ import ( "sync" "testing" + gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/pkg/gfi" ) @@ -29,6 +30,27 @@ func TestHardSuppressIssueLabelsIsCopyWithoutGFI(t *testing.T) { } } +func TestGFIArchivedWriteRecognizesOnlyArchived403(t *testing.T) { + c := NewClientForTest("http://example.com", "acme", []string{"widgets"}, testLogger()) + for _, tc := range []struct { + status int + message string + want bool + }{ + {http.StatusForbidden, "Resource not accessible by integration", false}, + {http.StatusInternalServerError, "Repository was archived", false}, + {http.StatusForbidden, "Repository was archived so is read-only.", true}, + } { + err := &gh.ErrorResponse{Response: &http.Response{StatusCode: tc.status}, Message: tc.message} + if got := c.ArchivedWrite("widgets", err); got != tc.want { + t.Fatalf("status=%d message=%q: got %v", tc.status, tc.message, got) + } + } + if !c.ArchivedWrite("ACME/Widgets", nil) || c.ArchivedWrite("acme/other", nil) { + t.Fatal("archive cache must be case-insensitive and repository-scoped") + } +} + func TestUpsertIssueMarkerComment(t *testing.T) { var mu sync.Mutex comments := []map[string]any{ From 7f6670c345894173cf41f2a319616d46ff648bc7 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 22:33:36 +0000 Subject: [PATCH 012/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.222.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.222.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-substrate-sandbox-runtime.md | 1 - changelog.d/changed-substrate-runtime-docs.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-substrate-sandbox-runtime.md delete mode 100644 changelog.d/changed-substrate-runtime-docs.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c95a8aeea6..cb85f26dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.222.0) + +### Added + +- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. + +### Changed + +- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. + ## 2026-11-11 (v5.221.1) ### Fixed diff --git a/changelog.d/added-substrate-sandbox-runtime.md b/changelog.d/added-substrate-sandbox-runtime.md deleted file mode 100644 index f9c190943b..0000000000 --- a/changelog.d/added-substrate-sandbox-runtime.md +++ /dev/null @@ -1 +0,0 @@ -- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. diff --git a/changelog.d/changed-substrate-runtime-docs.md b/changelog.d/changed-substrate-runtime-docs.md deleted file mode 100644 index fadfda564d..0000000000 --- a/changelog.d/changed-substrate-runtime-docs.md +++ /dev/null @@ -1 +0,0 @@ -- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. From 68eb32b0d615745f1dc8f2dc922d63170509b36b Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 15:37:36 -0700 Subject: [PATCH 013/223] =?UTF-8?q?=F0=9F=90=9B=20marketplace:=20resume=20?= =?UTF-8?q?CNCF=20catalogue=20seeding=20across=20hub=20restarts=20and=20le?= =?UTF-8?q?t=20spokes=20trigger=20refresh=20(#17773)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 marketplace: resume CNCF catalogue seeding across hub restarts and let spokes trigger refresh Root cause of the one-entry catalogue on hive.hivecommons.dev: GitHub tarballs lead with a pax_global_header entry, which Go's tar reader surfaces as a header. ExtractTarGz counted it as a second top-level name, returned dest instead of the wrapped "--/" directory, and ingest.ReadTree then saw every path with the wrapper prefix — no GitHub Actions workflow matched. Only tektoncd/pipeline got through, via the content-based Tekton sniff. Skip pax headers when extracting. Seeding is now resumable: at startup the hub ingests every corpus repo that is due (no record, record older than 24h or in an older format, error older than 1h) instead of only when the catalogue is empty, so a restart mid-seed continues and a flapping hub does not re-crawl fresh repos. Records carry a format version; bumping it (format 2 for the tarball fix) re-ingests everything on the next boot. The last run summary is persisted so status.lastRefresh survives restarts; status gains pendingRepos/lastRefreshErrors and ?verbose=1 (spoke or admin) lists the failing repos with their errors. POST /api/marketplace/refresh accepts the same credentials as publish (hub admin session or spoke heartbeat bearer + X-Hive-ID); spokes are rate-limited (one full run/hour, 10 ?repo= runs/min → 429 + Retry-After), admins are not. The spoke proxy passes 429/409 through with the hub's message. The crawl sleeps on 403/429 Retry-After (secondary limits) and logs progress every 25 repos. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson * 🐛 marketplace: update refresh tooltip in static-src and regenerate index.html Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/fixed-marketplace-seed-resume.md | 1 + src/docs/api-reference/hub-server.md | 10 +- src/docs/design/workflow-models.md | 46 ++- .../dashboard/api_workflows_marketplace.go | 13 +- .../api_workflows_marketplace_test.go | 26 +- .../js/config-workflows-marketplace.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/evidence/complexity/runner_test.go | 12 + src/pkg/evidence/complexity/tarball.go | 8 + src/pkg/hub/marketplace.go | 112 ++++-- src/pkg/hub/marketplace_refresh.go | 147 +++++++- src/pkg/hub/marketplace_store.go | 42 +++ src/pkg/hub/marketplace_test.go | 344 +++++++++++++++++- 13 files changed, 690 insertions(+), 75 deletions(-) create mode 100644 changelog.d/fixed-marketplace-seed-resume.md diff --git a/changelog.d/fixed-marketplace-seed-resume.md b/changelog.d/fixed-marketplace-seed-resume.md new file mode 100644 index 0000000000..5321431ab7 --- /dev/null +++ b/changelog.d/fixed-marketplace-seed-resume.md @@ -0,0 +1 @@ +- Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 5fd503f013..b79feb07b9 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -71,8 +71,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | | `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1802` | | `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1804` | -| `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:34` | -| `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:35` | -| `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:36` | -| `POST` | `/api/marketplace/refresh` | Hub admin | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}` | `pkg/hub/marketplace.go:37` | -| `GET` | `/api/marketplace/status` | Public | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary and counts by tier | `pkg/hub/marketplace.go:38` | +| `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | +| `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | +| `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | +| `POST` | `/api/marketplace/refresh` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}`. Spokes are rate-limited across all hives (one full run per hour, 10 `?repo=` runs per minute) → `429` + `Retry-After`; admins are not. `401` anonymous, `403` non-admin session or unknown hive | `pkg/hub/marketplace.go:39` | +| `GET` | `/api/marketplace/status` | Public; `?verbose=1` requires spoke heartbeat bearer + `X-Hive-ID` or Hub admin session | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, pendingRepos, lastRefreshErrors, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary (persisted across restarts) and counts by tier. `?verbose=1` adds `failing:[{repo, error, checkedAt}]` and `pending:[repo]` (corpus repos never ingested) | `pkg/hub/marketplace.go:40` | diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index d5e7bd53e5..10ed66c3ef 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -719,6 +719,7 @@ Storage is a directory under the hub data dir: workflows//.bundle.json immutable; refuse overwrite of name@version index.json derived; rebuilt from the bundles if missing refresh/__.json last ingest run per repository + last-refresh.json summary of the last completed run ``` Entries are keyed `name@version` plus hash. Publishing the identical bundle again is a @@ -734,8 +735,8 @@ Routes (see [hub-server.md](../api-reference/hub-server.md)): | `GET /api/marketplace/workflows` | public | index; `If-None-Match` → `304` | | `GET /api/marketplace/workflows/{name}/{version}/bundle` | public | one bundle, `Cache-Control: immutable` | | `POST /api/marketplace/workflows` | spoke bearer or hub admin | publish; hub re-verifies | -| `POST /api/marketplace/refresh[?repo=]` | hub admin | on-demand ingest | -| `GET /api/marketplace/status` | public | last refresh, counts by tier | +| `POST /api/marketplace/refresh[?repo=]` | spoke bearer or hub admin | on-demand ingest; spokes rate-limited (`429`) | +| `GET /api/marketplace/status[?verbose=1]` | public (`verbose`: spoke bearer or hub admin) | last refresh, counts by tier; `verbose` adds failing and pending repos | **Who may publish what.** A spoke authenticates with its heartbeat bearer bound to the `X-Hive-ID` header (the same per-hive key the heartbeat and NPS lanes verify) and may @@ -748,16 +749,37 @@ same-origin CSRF evidence. The hub never stores a bundle it did not itself `Veri falls back to it when no `projects.json` is given) plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch tarball through its GitHub identity, keeps only pipeline files (`ingest.ReadTree`), and -records `{sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → +records `{format, sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → `ingest.Lifecycle` → `bundle.Export` → publish as `ingested-YYYYMMDD.N`, tier `ingested-unreviewed`, with the source SHA in an annotation. Curated versions are never touched; `marketplaceNormaliseIndex` flags a curated entry `newerIngestAvailable` when a -later ingest of the same name exists. The crawl is sequential with a short pause between -repositories and honours `X-RateLimit-Remaining`/`Reset`. It runs nightly at -`HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE` (HH:MM UTC, default `03:00`, empty disables), on -demand, and once at hub start when the catalogue is empty and a GitHub token is present -(in a goroutine; startup is not blocked). `HIVE_HUB_MARKETPLACE_DISABLED` turns the -routes off (`404`). +later ingest of the same name exists. The crawl is sequential in sorted repository order +with a short pause between repositories, logs progress every 25, honours +`X-RateLimit-Remaining`/`Reset` and sleeps on a `403`/`429` with `Retry-After` (secondary +limits). It runs nightly at `HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE` (HH:MM UTC, default +`03:00`, empty disables) and on demand. `HIVE_HUB_MARKETPLACE_DISABLED` turns the routes off +(`404`). + +**Seeding is resumable.** Each repository's record is persisted as the crawl goes, so a +run interrupted by a hub restart (releases roll several times a day) loses nothing. At +start, after a short delay and when a GitHub token is present, the hub ingests every corpus +repository that is *due*: no record, a record older than the nightly interval (24 h) or in +an older `format`, or an error older than one hour. A hub that restarts repeatedly +therefore only re-crawls what is missing instead of hammering GitHub, and a catalogue that +seeded one entry before a roll fills in on the next boot. Bumping +`marketplaceRecordFormat` forces a full re-ingest when the ingest itself changes (format 2: +tarball extraction recognises git's `pax_global_header`, without which the wrapped +directory was not detected and almost every repository read as having no pipeline files). +The last run's summary is persisted (`last-refresh.json`) so `status.lastRefresh` survives a +restart. + +**Refresh auth.** `POST /api/marketplace/refresh` takes the same credentials as publish: a +hub-admin session (same-origin CSRF evidence) or a registered spoke's heartbeat bearer + +`X-Hive-ID`, so a hive owner's **Refresh catalogue** button works through the spoke proxy. +Spokes are rate-limited across all hives — one full run per hour, ten `?repo=` runs per +minute — and get `429` with `Retry-After`; admins are not limited. A run already in flight +is `409`. The spoke relays the hub's `401/403` as `403`, keeps `429`/`409`, and maps other +failures to `502`, always with the hub's message in the body. ### Spoke — import, publish, browse @@ -785,8 +807,10 @@ Routes (`pkg/dashboard/api_workflows_marketplace.go`): - `POST /api/marketplace/workflows/publish {name, version}` (owner) — export and POST to the hub with the spoke's heartbeat credential; `201 {published, hash, tier}`. Audit `workflow_publish`. -- `POST /api/marketplace/refresh[?repo=]` (owner) — forwards to the hub; a non-admin hive - surfaces the hub's `403`. Audit `workflow_marketplace_refresh`. +- `POST /api/marketplace/refresh[?repo=]` (owner) — forwards to the hub with the spoke's + heartbeat credential; the hub's `429` (rate limit, with `Retry-After`) and `409` (run in + progress) pass through with its message, `401/403` → `403`. Audit + `workflow_marketplace_refresh`. Imported models are pinned `name@version`; a newer catalogue version is a new library entry and **bindings never move automatically** — rebinding stays an explicit act on Settings → diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go index 98d2390836..b7836e8b2a 100644 --- a/src/pkg/dashboard/api_workflows_marketplace.go +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -450,8 +450,11 @@ func (s *Server) handleMarketplacePublish(w http.ResponseWriter, r *http.Request } // handleMarketplaceRefresh asks the hub to re-ingest its corpus (or one -// ?repo=). Owner on the spoke; the hub applies its own admin policy and may -// refuse, which is surfaced as 403. +// ?repo=). Owner on the spoke; the request carries the spoke's heartbeat +// bearer + X-Hive-ID, which the hub accepts for refresh the way it does +// for publish. The hub's error body is relayed verbatim: 401/403 → 403, +// 429 (hive rate limit) and 409 (run in progress) keep their status, and +// anything else → 502. func (s *Server) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !requireOwnerRole(w, r) { return @@ -482,6 +485,12 @@ func (s *Server) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: jsonError(w, "hub refused refresh: "+strings.TrimSpace(hubErrorMessage(body)), http.StatusForbidden) return + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusConflict: + if ra := resp.Header.Get("Retry-After"); ra != "" { + w.Header().Set("Retry-After", ra) + } + jsonError(w, "hub refresh not started: "+strings.TrimSpace(hubErrorMessage(body)), resp.StatusCode) + return case resp.StatusCode >= 300: jsonError(w, fmt.Sprintf("hub refresh failed (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(hubErrorMessage(body))), http.StatusBadGateway) return diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go index 875fe0b7d0..eed2d9897b 100644 --- a/src/pkg/dashboard/api_workflows_marketplace_test.go +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -108,6 +108,10 @@ func newFakeHub(t *testing.T) (*fakeHub, *httptest.Server) { _, _ = w.Write([]byte(`{"error":"nope"}`)) case r.Method == http.MethodPost && r.URL.Path == "/api/marketplace/refresh": h.refreshes.Add(1) + if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") || r.Header.Get(marketplaceHiveIDHeader) == "" { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + return + } w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"accepted":true,"repo":"` + r.URL.Query().Get("repo") + `"}`)) default: @@ -403,12 +407,18 @@ func TestMarketplaceRefreshProxy(t *testing.T) { t.Fatalf("refresh: %d %s", rec.Code, rec.Body.String()) } deny := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.URL.RawQuery, "boom") { + switch { + case strings.Contains(r.URL.RawQuery, "boom"): w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("plain text failure")) - return + case strings.Contains(r.URL.RawQuery, "slow"): + w.Header().Set("Retry-After", "42") + http.Error(w, `{"error":"full catalogue refresh was requested recently by a hive; try again in 42s"}`, http.StatusTooManyRequests) + case strings.Contains(r.URL.RawQuery, "busy"): + http.Error(w, `{"error":"marketplace: refresh already running"}`, http.StatusConflict) + default: + http.Error(w, `{"error":"admin only"}`, http.StatusForbidden) } - http.Error(w, `{"error":"admin only"}`, http.StatusForbidden) })) defer deny.Close() s.deps.Config.Workflow.Marketplace.URL = deny.URL @@ -416,6 +426,16 @@ func TestMarketplaceRefreshProxy(t *testing.T) { if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "admin only") { t.Fatalf("denied: %d %s", rec.Code, rec.Body.String()) } + // Hub rate limit and busy keep their status and message so the toast + // tells the owner when to retry. + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=slow/x", nil, true) + if rec.Code != http.StatusTooManyRequests || rec.Header().Get("Retry-After") != "42" || !strings.Contains(rec.Body.String(), "try again in 42s") { + t.Fatalf("rate limited: %d %s retry-after=%q", rec.Code, rec.Body.String(), rec.Header().Get("Retry-After")) + } + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=busy/x", nil, true) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "already running") { + t.Fatalf("busy: %d %s", rec.Code, rec.Body.String()) + } rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=boom", nil, true) if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "plain text failure") { t.Fatalf("hub 500: %d %s", rec.Code, rec.Body.String()) diff --git a/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js b/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js index fbc7d75f3e..180baf32bb 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js @@ -100,7 +100,7 @@ `` + `${shown.length} of ${entries.length}` + `` + - (owner ? `` : '') + + (owner ? `` : '') + ''; let body; if (!st.loaded && st.loading) body = '

Loading the catalogue…

'; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index db978b8760..92bfc42a08 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -65926,7 +65926,7 @@

Agent Watchdog

`` + `${shown.length} of ${entries.length}` + `` + - (owner ? `` : '') + + (owner ? `` : '') + ''; let body; if (!st.loaded && st.loading) body = '

Loading the catalogue…

'; diff --git a/src/pkg/evidence/complexity/runner_test.go b/src/pkg/evidence/complexity/runner_test.go index 9fbacb52b7..a91d5269e6 100644 --- a/src/pkg/evidence/complexity/runner_test.go +++ b/src/pkg/evidence/complexity/runner_test.go @@ -393,6 +393,18 @@ func TestExtractTarGz(t *testing.T) { t.Error("symlink must be skipped") } + // git archive leads with a pax_global_header entry; it must not be + // mistaken for a second top-level name. + pax := tarGz(t, map[string]string{"o-r-abc1234/p.go": "package p\n"}, + tar.Header{Name: "pax_global_header", Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": "abc1234"}}) + d2 := t.TempDir() + if top, err := ExtractTarGz(bytes.NewReader(pax), d2, 0); err != nil || top != filepath.Join(d2, "o-r-abc1234") { + t.Errorf("pax header: top=%s err=%v", top, err) + } + if _, err := os.Stat(filepath.Join(d2, "pax_global_header")); !errors.Is(err, os.ErrNotExist) { + t.Error("pax_global_header must not be written") + } + // Path traversal is refused. bad := tarGz(t, map[string]string{"../escape.go": "x"}) if _, err := ExtractTarGz(bytes.NewReader(bad), t.TempDir(), 0); err == nil { diff --git a/src/pkg/evidence/complexity/tarball.go b/src/pkg/evidence/complexity/tarball.go index 056d3a626c..2a3814cf93 100644 --- a/src/pkg/evidence/complexity/tarball.go +++ b/src/pkg/evidence/complexity/tarball.go @@ -46,6 +46,14 @@ func ExtractTarGz(r io.Reader, dest string, maxBytes int64) (string, error) { if err != nil { return "", err } + // git archive (and so GitHub's tarball endpoint) leads with a + // "pax_global_header" entry carrying the commit SHA; the reader + // surfaces it as a header. It is not a tree entry and must not + // count as a second top-level name, or the wrapped directory is + // never detected and callers read paths with the wrapper prefix. + if hdr.Typeflag == tar.TypeXGlobalHeader || hdr.Typeflag == tar.TypeXHeader { + continue + } name := filepath.Clean(filepath.FromSlash(hdr.Name)) if name == "." || name == "" { continue diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 3eb24ea673..73e5196ae1 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "time" @@ -17,8 +18,9 @@ import ( // GET /api/marketplace/workflows index (ETag / 304) // GET /api/marketplace/workflows/{name}/{version}/bundle one bundle // POST /api/marketplace/workflows publish (spoke bearer or admin) -// POST /api/marketplace/refresh[?repo=] on-demand ingest (admin) -// GET /api/marketplace/status last refresh + counts by tier +// POST /api/marketplace/refresh[?repo=] on-demand ingest (spoke bearer or admin) +// GET /api/marketplace/status[?verbose=1] last refresh + counts by tier +// (verbose: failing/pending repos; spoke bearer or admin) // // Tier policy: a publisher may assert `local`, `local-published` or // `ingested-unreviewed`; `curated` and `built-in` require a hub admin @@ -34,7 +36,7 @@ func (s *HubServer) registerMarketplaceRoutes() { s.mux.HandleFunc("GET /api/marketplace/workflows", s.handleMarketplaceIndex) s.mux.HandleFunc("GET /api/marketplace/workflows/{name}/{version}/bundle", s.handleMarketplaceBundle) s.mux.HandleFunc("POST /api/marketplace/workflows", s.handleMarketplacePublish) - s.mux.HandleFunc("POST /api/marketplace/refresh", s.requireAdmin(s.handleMarketplaceRefresh)) + s.mux.HandleFunc("POST /api/marketplace/refresh", s.handleMarketplaceRefresh) s.mux.HandleFunc("GET /api/marketplace/status", s.handleMarketplaceStatus) } @@ -207,18 +209,37 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ // handleMarketplaceRefresh runs an on-demand ingest in the background and // answers 202 immediately; ?repo= limits it to one repository (and runs // synchronously, since a single repo is quick enough to await). +// +// Callers are the same as publish: a hub admin session or a registered +// spoke with its heartbeat bearer + X-Hive-ID, so a hive owner's "Refresh +// catalogue" button works through the spoke. Spokes are rate-limited (one +// full run per marketplaceFullRefreshInterval across all spokes, +// marketplaceRepoRefreshBurst single-repo runs per minute) → 429 with +// Retry-After; admins are not. func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return } + who, code, msg := s.marketplaceAuthenticate(r) + if code != 0 { + marketplaceJSONError(w, code, msg) + return + } repo := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("repo"))) - user := s.getRealAuthUser(r) + rf := s.refresher() + now := rf.now().UTC() if repo != "" { if !strings.Contains(repo, "/") { marketplaceJSONError(w, http.StatusBadRequest, "repo must be owner/name") return } - res, err := s.runMarketplaceRefresh(r.Context(), []string{repo}, "on-demand:"+user) + if !who.admin { + if ok, wait := rf.allowRepoRefresh(now); !ok { + marketplaceTooMany(w, wait, "single-repository refresh") + return + } + } + res, err := s.runMarketplaceRefresh(r.Context(), []string{repo}, "on-demand:"+who.name) if errors.Is(err, errMarketplaceRefreshBusy) { marketplaceJSONError(w, http.StatusConflict, err.Error()) return @@ -227,12 +248,11 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) return } - s.logger.Info("marketplace refresh (single repo)", "by", user, "repo", repo, "published", res.Published, "errors", res.Errors) + s.logger.Info("marketplace refresh (single repo)", "by", who.name, "repo", repo, "published", res.Published, "errors", res.Errors) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": true, "repo": repo, "result": res}) return } - rf := s.refresher() rf.mu.Lock() busy := rf.running rf.mu.Unlock() @@ -240,9 +260,15 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusConflict, errMarketplaceRefreshBusy.Error()) return } - s.logger.Info("marketplace refresh requested", "by", user) + if !who.admin { + if ok, wait := rf.allowFullRefresh(now); !ok { + marketplaceTooMany(w, wait, "full catalogue refresh") + return + } + } + s.logger.Info("marketplace refresh requested", "by", who.name) go func() { - if _, err := s.runMarketplaceRefresh(s.marketplaceBackgroundContext(), nil, "on-demand:"+user); err != nil { + if _, err := s.runMarketplaceRefresh(s.marketplaceBackgroundContext(), nil, "on-demand:"+who.name); err != nil { s.logger.Warn("marketplace on-demand refresh skipped", "error", err) } }() @@ -251,11 +277,39 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ _ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": true, "repos": len(marketplaceRepos())}) } +// marketplaceTooMany answers 429 with Retry-After and an honest message: +// the spoke relays the body into the operator's toast. +func marketplaceTooMany(w http.ResponseWriter, wait time.Duration, what string) { + secs := int(wait.Round(time.Second) / time.Second) + if secs < 1 { + secs = 1 + } + w.Header().Set("Retry-After", strconv.Itoa(secs)) + marketplaceJSONError(w, http.StatusTooManyRequests, fmt.Sprintf("%s was requested recently by a hive; try again in %s (hub admins are not limited)", what, (time.Duration(secs)*time.Second).String())) +} + +// marketplaceFailingRepo is one verbose-status row for a repo whose last +// ingest failed. +type marketplaceFailingRepo struct { + Repo string `json:"repo"` + Error string `json:"error"` + CheckedAt time.Time `json:"checkedAt"` +} + // handleMarketplaceStatus reports the last refresh and tier counts. +// ?verbose=1 (spoke bearer or admin) adds the failing repositories with +// their error strings and the repositories still awaiting a first ingest. func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return } + verbose := r.URL.Query().Get("verbose") != "" + if verbose { + if _, code, msg := s.marketplaceAuthenticate(r); code != 0 { + marketplaceJSONError(w, code, msg) + return + } + } idx, _, etag, err := s.marketplace().Index() if err != nil { marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) @@ -272,31 +326,45 @@ func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Reque rf.mu.Unlock() records := s.marketplace().refreshRecords() var lastChecked time.Time - failing := 0 + failing := []marketplaceFailingRepo{} for _, rec := range records { if rec.CheckedAt.After(lastChecked) { lastChecked = rec.CheckedAt } if rec.Error != "" { - failing++ + failing = append(failing, marketplaceFailingRepo{Repo: rec.Repo, Error: rec.Error, CheckedAt: rec.CheckedAt}) } } + pending := s.marketplacePendingRepos() + lastErrors := 0 + if last != nil { + lastErrors = len(last.Errors) + } out := map[string]interface{}{ - "entries": len(idx.Entries), - "byTier": byTier, - "etag": etag, - "generatedAt": idx.GeneratedAt, - "corpusRepos": len(marketplaceRepos()), - "trackedRepos": len(records), - "failingRepos": failing, - "refreshing": running, - "schedule": marketplaceSchedule(), - "lastRefresh": last, - "lastCheckedAt": nil, + "entries": len(idx.Entries), + "byTier": byTier, + "etag": etag, + "generatedAt": idx.GeneratedAt, + "corpusRepos": len(marketplaceRepos()), + "trackedRepos": len(records), + "failingRepos": len(failing), + "pendingRepos": len(pending), + "lastRefreshErrors": lastErrors, + "refreshing": running, + "schedule": marketplaceSchedule(), + "lastRefresh": last, + "lastCheckedAt": nil, } if !lastChecked.IsZero() { out["lastCheckedAt"] = lastChecked } + if verbose { + if pending == nil { + pending = []string{} + } + out["failing"] = failing + out["pending"] = pending + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(out) } diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index d9951a30b4..ed164784ce 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -49,6 +49,22 @@ const ( marketplaceRateMaxWait = 10 * time.Minute marketplaceMaxTarball = 256 << 20 marketplaceSeedDelay = 30 * time.Second + // marketplaceStaleAfter is how old a repo's refresh record may be + // before the startup pass re-ingests it: the nightly interval, so a + // hub that restarts repeatedly only re-crawls what the nightly missed. + marketplaceStaleAfter = 24 * time.Hour + // marketplaceRetryErrorAfter is how soon the startup pass retries a + // repository whose last record is an error. + marketplaceRetryErrorAfter = time.Hour + // marketplaceProgressEvery is how many repos between progress log lines. + marketplaceProgressEvery = 25 + // marketplaceFullRefreshInterval bounds on-demand full runs from + // non-admin callers (spokes): one per interval across all callers. + marketplaceFullRefreshInterval = time.Hour + // marketplaceRepoRefreshBurst / Window bound on-demand single-repo + // runs from non-admin callers. + marketplaceRepoRefreshBurst = 10 + marketplaceRepoRefreshWindow = time.Minute ) // marketplaceRepoPause keeps the crawl polite: one repo at a time with a @@ -84,6 +100,39 @@ type marketplaceRefresher struct { // bgCtx bounds on-demand runs started from a request; set by // StartMarketplaceRefresh so they stop with the other pollers. bgCtx context.Context + // lastFullRequest and repoRequests back the on-demand rate limits. + lastFullRequest time.Time + repoRequests []time.Time +} + +// allowFullRefresh reports whether a non-admin full refresh may start now +// and, when not, how long until one may. Callers hold no lock. +func (rf *marketplaceRefresher) allowFullRefresh(now time.Time) (bool, time.Duration) { + rf.mu.Lock() + defer rf.mu.Unlock() + if wait := marketplaceFullRefreshInterval - now.Sub(rf.lastFullRequest); !rf.lastFullRequest.IsZero() && wait > 0 { + return false, wait + } + rf.lastFullRequest = now + return true, 0 +} + +// allowRepoRefresh is the sliding-window limit for non-admin ?repo= runs. +func (rf *marketplaceRefresher) allowRepoRefresh(now time.Time) (bool, time.Duration) { + rf.mu.Lock() + defer rf.mu.Unlock() + kept := rf.repoRequests[:0] + for _, t := range rf.repoRequests { + if now.Sub(t) < marketplaceRepoRefreshWindow { + kept = append(kept, t) + } + } + rf.repoRequests = kept + if len(kept) >= marketplaceRepoRefreshBurst { + return false, marketplaceRepoRefreshWindow - now.Sub(kept[0]) + } + rf.repoRequests = append(kept, now) + return true, 0 } // marketplaceBackgroundContext is the pollers' context when running, else @@ -106,6 +155,9 @@ func (s *HubServer) marketplace() *marketplaceStore { if s.marketplaceRefresher == nil { s.marketplaceRefresher = &marketplaceRefresher{fetch: s.marketplaceGitHubFetcher, now: time.Now} } + if s.marketplaceRefresher.last == nil { + s.marketplaceRefresher.last = s.marketplaceStore.loadLastRefresh() + } }) return s.marketplaceStore } @@ -168,6 +220,7 @@ func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, t repos = marketplaceRepos() } res := &marketplaceRefreshResult{StartedAt: rf.now().UTC(), Trigger: trigger, Published: []string{}, Errors: []string{}} + s.logger.Info("marketplace refresh started", "trigger", trigger, "repos", len(repos)) for i, repo := range repos { if ctx.Err() != nil { res.Errors = append(res.Errors, "cancelled: "+ctx.Err().Error()) @@ -192,11 +245,17 @@ func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, t default: res.Published = append(res.Published, outcome) } + if res.Repos%marketplaceProgressEvery == 0 && res.Repos < len(repos) { + s.logger.Info("marketplace refresh progress", "trigger", trigger, "done", res.Repos, "total", len(repos), "published", len(res.Published), "errors", len(res.Errors)) + } } res.FinishedAt = rf.now().UTC() rf.mu.Lock() rf.last = res rf.mu.Unlock() + if err := st.saveLastRefresh(res); err != nil { + s.logger.Warn("marketplace refresh: could not persist summary", "error", err) + } s.logger.Info("marketplace refresh finished", "trigger", trigger, "repos", res.Repos, "published", len(res.Published), "unchanged", res.Unchanged, "skipped", res.Skipped, "errors", len(res.Errors)) return res, nil } @@ -213,7 +272,7 @@ func (s *HubServer) refreshOneRepo(ctx context.Context, st *marketplaceStore, rf } hash := marketplaceFilesHash(files) rec := marketplaceRefreshRecord{Repo: repo, SourceSHA: sha, FilesHash: hash, Files: len(files), CheckedAt: rf.now().UTC()} - if prev, ok := st.refreshRecord(repo); ok && prev.FilesHash == hash && prev.Error == "" { + if prev, ok := st.refreshRecord(repo); ok && prev.Format == marketplaceRecordFormat && prev.FilesHash == hash && prev.Error == "" { rec.Published = prev.Published _ = st.saveRefreshRecord(rec) return "", nil @@ -313,6 +372,7 @@ func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) ( return nil, "", err } defer resp.Body.Close() + marketplaceHonourRateLimit(ctx, resp.StatusCode, resp.Header) if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) return nil, "", fmt.Errorf("tarball: %s", resp.Status) @@ -334,8 +394,9 @@ func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) ( } // marketplaceGitHubJSON GETs a GitHub API resource, honouring rate-limit -// headers: when the remaining budget is under marketplaceRateFloor it -// sleeps until the reset (bounded) before returning. +// headers: when the remaining budget is under marketplaceRateFloor, or the +// reply is a 403/429 carrying Retry-After (secondary limit), it sleeps +// (bounded) before returning so the next repo does not hit the same wall. func marketplaceGitHubJSON(ctx context.Context, client *http.Client, url string, out interface{}) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -348,7 +409,7 @@ func marketplaceGitHubJSON(ctx context.Context, client *http.Client, url string, return err } defer resp.Body.Close() - marketplaceHonourRateLimit(ctx, resp.Header) + marketplaceHonourRateLimit(ctx, resp.StatusCode, resp.Header) if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) return fmt.Errorf("%s: %s", url, resp.Status) @@ -364,7 +425,20 @@ var marketplaceSleep = func(ctx context.Context, d time.Duration) { } } -func marketplaceHonourRateLimit(ctx context.Context, h http.Header) { +func marketplaceHonourRateLimit(ctx context.Context, status int, h http.Header) { + if status == http.StatusForbidden || status == http.StatusTooManyRequests { + // Secondary rate limits answer 403/429 with Retry-After seconds and + // a non-zero X-RateLimit-Remaining, so the floor check below would + // not catch them. + if secs, err := strconv.Atoi(strings.TrimSpace(h.Get("Retry-After"))); err == nil && secs > 0 { + wait := time.Duration(secs) * time.Second + if wait > marketplaceRateMaxWait { + wait = marketplaceRateMaxWait + } + marketplaceSleep(ctx, wait) + return + } + } rem, err := strconv.Atoi(strings.TrimSpace(h.Get("X-RateLimit-Remaining"))) if err != nil || rem >= marketplaceRateFloor { return @@ -412,9 +486,12 @@ func marketplaceSchedule() string { return strings.TrimSpace(v) } -// StartMarketplaceRefresh runs the nightly ingest until ctx is cancelled, -// seeding the catalogue once at start when it is empty and the hub has a -// GitHub token. Started from StartBackgroundPollers. +// StartMarketplaceRefresh runs the nightly ingest until ctx is cancelled. +// At start (after marketplaceSeedDelay) it ingests every corpus repository +// that is due — no refresh record, a record older than marketplaceStaleAfter +// or in an older format, or an error older than marketplaceRetryErrorAfter — +// so a seed interrupted by a hub restart resumes where it stopped instead of +// waiting for the nightly. Started from StartBackgroundPollers. func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { if marketplaceDisabled() { s.logger.Info("marketplace disabled", "env", hubMarketplaceDisableEnv) @@ -426,11 +503,11 @@ func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { rf.mu.Unlock() schedule := marketplaceSchedule() s.logger.Info("marketplace refresh started", "schedule_utc", schedule, "corpus_repos", len(marketplaceRepos())) - if s.marketplaceShouldSeed() { - s.logger.Warn("marketplace catalogue is empty: seeding from the CNCF corpus in the background", "delay", marketplaceSeedDelay.String()) + if due := s.marketplaceDueRepos(); len(due) > 0 { + s.logger.Warn("marketplace catalogue has repositories due for ingest: resuming in the background", "due", len(due), "delay", marketplaceSeedDelay.String()) marketplaceSleep(ctx, marketplaceSeedDelay) if ctx.Err() == nil { - if _, err := s.runMarketplaceRefresh(ctx, nil, "seed"); err != nil { + if _, err := s.runMarketplaceRefresh(ctx, due, "seed"); err != nil { s.logger.Warn("marketplace seed did not run", "error", err) } } @@ -452,12 +529,48 @@ func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { } } -// marketplaceShouldSeed is true when the catalogue has no entries and the -// hub can reach GitHub with a token. -func (s *HubServer) marketplaceShouldSeed() bool { +// marketplaceRepoDue reports whether a repository needs ingesting at +// startup given its last record (ok=false when there is none). +func marketplaceRepoDue(rec marketplaceRefreshRecord, ok bool, now time.Time) bool { + switch { + case !ok: + return true + case rec.Format != marketplaceRecordFormat: + return true + case rec.Error != "": + return now.Sub(rec.CheckedAt) >= marketplaceRetryErrorAfter + default: + return now.Sub(rec.CheckedAt) >= marketplaceStaleAfter + } +} + +// marketplaceDueRepos is the sorted subset of marketplaceRepos the startup +// pass should ingest, or nil when the hub has no GitHub token (nothing can +// be fetched, so nothing is due). +func (s *HubServer) marketplaceDueRepos() []string { if hubGitHubAuthTokenForRequest(context.Background()) == "" { - return false + return nil + } + st := s.marketplace() + now := s.refresher().now().UTC() + var due []string + for _, repo := range marketplaceRepos() { + rec, ok := st.refreshRecord(repo) + if marketplaceRepoDue(rec, ok, now) { + due = append(due, repo) + } + } + return due +} + +// marketplacePendingRepos counts corpus repositories with no refresh record. +func (s *HubServer) marketplacePendingRepos() []string { + st := s.marketplace() + var pending []string + for _, repo := range marketplaceRepos() { + if _, ok := st.refreshRecord(repo); !ok { + pending = append(pending, repo) + } } - idx, _, _, err := s.marketplace().Index() - return err == nil && len(idx.Entries) == 0 + return pending } diff --git a/src/pkg/hub/marketplace_store.go b/src/pkg/hub/marketplace_store.go index 5864ca561d..5f83eb1775 100644 --- a/src/pkg/hub/marketplace_store.go +++ b/src/pkg/hub/marketplace_store.go @@ -24,6 +24,7 @@ import ( // workflows//.bundle.json immutable published bundles // index.json derived catalogue index // refresh/__.json last ingest run per corpus repo +// last-refresh.json summary of the last completed run // // Entries are keyed name@version and never overwritten: a republish with the // same key and hash is a no-op, a different hash is a 409. Curated versions @@ -302,9 +303,18 @@ func marketplaceNormaliseIndex(idx *bundle.Index) { } } +// marketplaceRecordFormat versions the refresh record. Bumping it makes +// every existing record due at the next startup pass, which is how a fix to +// the ingest itself (not a change in the repository) gets re-crawled: +// +// 2 — tarball extraction recognises pax_global_header, so the pipeline +// files are actually read (format 1 recorded 0 files for most repos). +const marketplaceRecordFormat = 2 + // marketplaceRefreshRecord is what one ingest run remembers about a repo. type marketplaceRefreshRecord struct { Repo string `json:"repo"` + Format int `json:"format,omitempty"` SourceSHA string `json:"sourceSHA"` FilesHash string `json:"filesHash"` Files int `json:"files"` @@ -334,6 +344,7 @@ func (st *marketplaceStore) refreshRecord(repo string) (marketplaceRefreshRecord } func (st *marketplaceStore) saveRefreshRecord(rec marketplaceRefreshRecord) error { + rec.Format = marketplaceRecordFormat data, err := json.MarshalIndent(rec, "", " ") if err != nil { return err @@ -343,6 +354,37 @@ func (st *marketplaceStore) saveRefreshRecord(rec marketplaceRefreshRecord) erro return marketplaceWriteAtomic(filepath.Join(st.refreshDir(), marketplaceRepoSlug(rec.Repo)+".json"), data) } +func (st *marketplaceStore) lastRefreshPath() string { + return filepath.Join(st.root, "last-refresh.json") +} + +// saveLastRefresh persists a run summary so /api/marketplace/status can +// report it across hub restarts. +func (st *marketplaceStore) saveLastRefresh(res *marketplaceRefreshResult) error { + data, err := json.MarshalIndent(res, "", " ") + if err != nil { + return err + } + st.mu.Lock() + defer st.mu.Unlock() + return marketplaceWriteAtomic(st.lastRefreshPath(), data) +} + +// loadLastRefresh returns the persisted run summary, or nil when none. +func (st *marketplaceStore) loadLastRefresh() *marketplaceRefreshResult { + st.mu.RLock() + defer st.mu.RUnlock() + data, err := os.ReadFile(st.lastRefreshPath()) + if err != nil { + return nil + } + var res marketplaceRefreshResult + if json.Unmarshal(data, &res) != nil || res.StartedAt.IsZero() { + return nil + } + return &res +} + // refreshRecords lists every recorded run. func (st *marketplaceStore) refreshRecords() []marketplaceRefreshRecord { st.mu.RLock() diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index 5a0230dacf..185a7596a6 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -524,11 +525,13 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { } s := marketplaceTestHub(t, fetch) - // Non-admin / anon are refused by requireAdmin. - rec := mktDo(s, httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh?repo=a/b", nil)) - if rec.Code != 401 && rec.Code != 403 { - t.Fatalf("anon refresh: %d", rec.Code) + // Anon is refused. + anon := httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh?repo=a/b", nil) + anon.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, anon); rec.Code != 401 { + t.Fatalf("anon refresh: %d %s", rec.Code, rec.Body.String()) } + var rec *httptest.ResponseRecorder admin := func(target string) *http.Request { req := httptest.NewRequest(http.MethodPost, target, nil) req.AddCookie(testAuthCookie(hubAdminUsername)) @@ -588,12 +591,298 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if status["entries"].(float64) != 1 || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { t.Fatalf("status body: %s", rec.Body.String()) } + if status["pendingRepos"].(float64) != 0 || status["lastRefreshErrors"].(float64) != 0 { + t.Fatalf("pending/lastErrors: %s", rec.Body.String()) + } + if _, ok := status["failing"]; ok { + t.Fatal("non-verbose status must not list failing repos") + } byTier := status["byTier"].(map[string]interface{}) if byTier[bundle.TierIngestedUnreviewed].(float64) != 1 { t.Fatalf("byTier: %v", byTier) } } +// Resume after restart: a store with a record for one repo and a published +// entry must still trigger the startup pass for the rest, and a second +// start with everything fresh must not crawl again. +func TestMarketplaceSeedResumesAfterRestart(t *testing.T) { + t.Setenv(hubGitHubTokenEnv, "pat-token") + oldDelay := marketplaceSleep + marketplaceSleep = func(ctx context.Context, d time.Duration) { + if d != marketplaceSeedDelay { + <-ctx.Done() // the nightly park + } + } + t.Cleanup(func() { marketplaceSleep = oldDelay }) + + var mu sync.Mutex + fetched := map[string]int{} + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + mu.Lock() + fetched[repo]++ + mu.Unlock() + if repo == "konflux-ci/konflux-ci" { + return mktFixtureTree(t, "konflux-ci__konflux-ci"), "k1", nil + } + if repo == "broken/repo" { + return nil, "", errors.New("tarball: 404 Not Found") + } + return map[string][]byte{}, "none", nil + } + s := marketplaceTestHub(t, fetch) + t.Setenv(hubMarketplaceExtraReposEnv, "konflux-ci/konflux-ci,broken/repo") + all := marketplaceRepos() + + // "Before the restart": one repo ingested and published, nothing else. + if _, err := s.runMarketplaceRefresh(context.Background(), []string{"konflux-ci/konflux-ci"}, "seed"); err != nil { + t.Fatal(err) + } + if idx, _, _, _ := s.marketplace().Index(); len(idx.Entries) != 1 { + t.Fatalf("precondition: %d entries", len(idx.Entries)) + } + due := s.marketplaceDueRepos() + if len(due) != len(all)-1 { + t.Fatalf("due after partial seed: %d, want %d", len(due), len(all)-1) + } + for _, r := range due { + if r == "konflux-ci/konflux-ci" { + t.Fatal("freshly ingested repo must not be due") + } + } + + // "After the restart": StartMarketplaceRefresh resumes the rest. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { s.StartMarketplaceRefresh(ctx); close(done) }() + rf := s.refresher() + testutil.Eventually(t, 10*time.Second, func() bool { + rf.mu.Lock() + defer rf.mu.Unlock() + return rf.last != nil && rf.last.Trigger == "seed" && rf.last.Repos == len(all)-1 + }, "startup pass did not resume the remaining repos") + cancel() + <-done + mu.Lock() + n := fetched["konflux-ci/konflux-ci"] + mu.Unlock() + if n != 1 { + t.Fatalf("konflux fetched %d times; the resume must skip fresh records", n) + } + if len(s.marketplace().refreshRecords()) != len(all) { + t.Fatalf("records after resume: %d, want %d", len(s.marketplace().refreshRecords()), len(all)) + } + + // Everything fresh except the failing repo, which is retried only after + // marketplaceRetryErrorAfter. + if due := s.marketplaceDueRepos(); len(due) != 0 { + t.Fatalf("fresh records must not be due: %v", due) + } + rec, _ := s.marketplace().refreshRecord("broken/repo") + rec.CheckedAt = rec.CheckedAt.Add(-marketplaceRetryErrorAfter) + _ = s.marketplace().saveRefreshRecord(rec) + if due := s.marketplaceDueRepos(); len(due) != 1 || due[0] != "broken/repo" { + t.Fatalf("stale error must be due: %v", due) + } + + // Old-format records are due regardless of age. + now := rf.now() + if !marketplaceRepoDue(marketplaceRefreshRecord{Format: 1, CheckedAt: now}, true, now) { + t.Error("old format must be due") + } + if marketplaceRepoDue(marketplaceRefreshRecord{Format: marketplaceRecordFormat, CheckedAt: now.Add(-marketplaceStaleAfter + time.Minute)}, true, now) { + t.Error("fresh record must not be due") + } + if !marketplaceRepoDue(marketplaceRefreshRecord{Format: marketplaceRecordFormat, CheckedAt: now.Add(-marketplaceStaleAfter)}, true, now) { + t.Error("stale record must be due") + } +} + +// The last-run summary survives a restart (a fresh refresher over the same +// store), and the status route reports it with the error count. +func TestMarketplaceLastRefreshPersists(t *testing.T) { + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + if repo == "broken/repo" { + return nil, "", errors.New("tarball: 404 Not Found") + } + return map[string][]byte{}, "none", nil + } + s := marketplaceTestHub(t, fetch) + res, err := s.runMarketplaceRefresh(context.Background(), []string{"ok/repo", "broken/repo"}, "nightly") + if err != nil || len(res.Errors) != 1 { + t.Fatalf("run: %v %+v", err, res) + } + + // Simulate a restart: a new HubServer over the same marketplace dir. + s2 := newHandlerHub() + s2.marketplaceRefresher = &marketplaceRefresher{fetch: fetch, now: time.Now} + rf := s2.refresher() + rf.mu.Lock() + last := rf.last + rf.mu.Unlock() + if last == nil || last.Trigger != "nightly" || last.Repos != 2 || len(last.Errors) != 1 || !last.StartedAt.Equal(res.StartedAt) { + t.Fatalf("persisted last refresh: %+v", last) + } + + rec := mktDo(s2, httptest.NewRequest(http.MethodGet, "/api/marketplace/status", nil)) + var status map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status["lastRefresh"] == nil || status["lastRefreshErrors"].(float64) != 1 || status["failingRepos"].(float64) != 1 { + t.Fatalf("status after restart: %s", rec.Body.String()) + } + // Corrupt or missing summary → nil, not a crash. + st := s2.marketplace() + if err := os.WriteFile(st.lastRefreshPath(), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if st.loadLastRefresh() != nil { + t.Error("corrupt summary must load as nil") + } + if err := os.Remove(st.lastRefreshPath()); err != nil { + t.Fatal(err) + } + if st.loadLastRefresh() != nil { + t.Error("missing summary must load as nil") + } +} + +// Auth matrix for POST /api/marketplace/refresh: anon 401, spoke bearer ok +// (rate-limited), admin ok and unlimited; and the verbose status gate. +func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + if repo == "broken/repo" { + return nil, "", errors.New("tarball: 404 Not Found") + } + return map[string][]byte{}, "none", nil + } + s := marketplaceTestHub(t, fetch) + clock := s.refresher().now() + s.refresher().now = func() time.Time { return clock } + t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo") + + spoke := func(target string) *http.Request { + req := httptest.NewRequest(http.MethodPost, target, nil) + req.Header.Set("Authorization", heartbeatBearer(testHubSecret, mktTestHive)) + req.Header.Set(marketplaceHiveIDHeader, mktTestHive) + return req + } + admin := func(method, target string) *http.Request { + req := httptest.NewRequest(method, target, nil) + req.AddCookie(testAuthCookie(hubAdminUsername)) + req.Header.Set("Origin", "http://localhost") + return req + } + + // Anon. + anon := httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh", nil) + anon.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, anon); rec.Code != 401 { + t.Fatalf("anon: %d %s", rec.Code, rec.Body.String()) + } + // Non-admin session. + user := httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh", nil) + user.AddCookie(testAuthCookie("someuser")) + user.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, user); rec.Code != 403 { + t.Fatalf("non-admin session: %d", rec.Code) + } + // Spoke with a wrong bearer. + bad := spoke("/api/marketplace/refresh") + bad.Header.Set("Authorization", heartbeatBearer("wrong", mktTestHive)) + if rec := mktDo(s, bad); rec.Code != 401 { + t.Fatalf("bad bearer: %d", rec.Code) + } + + // Spoke single repo: ok, then burst-limited. + rec := mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) + if rec.Code != 200 || !strings.Contains(rec.Body.String(), "404 Not Found") { + t.Fatalf("spoke single: %d %s", rec.Code, rec.Body.String()) + } + for i := 1; i < marketplaceRepoRefreshBurst; i++ { + if rec = mktDo(s, spoke("/api/marketplace/refresh?repo=ok/repo")); rec.Code != 200 { + t.Fatalf("spoke single #%d: %d", i, rec.Code) + } + } + rec = mktDo(s, spoke("/api/marketplace/refresh?repo=ok/repo")) + if rec.Code != 429 || rec.Header().Get("Retry-After") == "" { + t.Fatalf("spoke burst: %d %s retry-after=%q", rec.Code, rec.Body.String(), rec.Header().Get("Retry-After")) + } + // Admin single repo is not limited. + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh?repo=ok/repo")); rec.Code != 200 { + t.Fatalf("admin single while spoke limited: %d", rec.Code) + } + // Window passes → spoke allowed again. + clock = clock.Add(marketplaceRepoRefreshWindow) + if rec = mktDo(s, spoke("/api/marketplace/refresh?repo=ok/repo")); rec.Code != 200 { + t.Fatalf("spoke after window: %d", rec.Code) + } + + // Spoke full run: 202 once per hour across callers; admin unaffected. + rf := s.refresher() + waitIdle := func() { + testutil.Eventually(t, 5*time.Second, func() bool { + rf.mu.Lock() + defer rf.mu.Unlock() + return !rf.running + }, "background refresh did not finish") + } + rec = mktDo(s, spoke("/api/marketplace/refresh")) + if rec.Code != 202 { + t.Fatalf("spoke full: %d %s", rec.Code, rec.Body.String()) + } + waitIdle() + rf.mu.Lock() + trigger := rf.last.Trigger + rf.mu.Unlock() + if trigger != "on-demand:hive:"+mktTestHive { + t.Fatalf("trigger = %q", trigger) + } + rec = mktDo(s, spoke("/api/marketplace/refresh")) + if rec.Code != 429 || !strings.Contains(rec.Body.String(), "try again in") { + t.Fatalf("spoke full again: %d %s", rec.Code, rec.Body.String()) + } + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh")); rec.Code != 202 { + t.Fatalf("admin full while spoke limited: %d %s", rec.Code, rec.Body.String()) + } + waitIdle() + clock = clock.Add(marketplaceFullRefreshInterval) + if rec = mktDo(s, spoke("/api/marketplace/refresh")); rec.Code != 202 { + t.Fatalf("spoke full after interval: %d", rec.Code) + } + waitIdle() + + // Verbose status: anon 401, spoke and admin get the failing list. + if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil)); rec.Code != 401 { + t.Fatalf("anon verbose: %d", rec.Code) + } + sv := httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil) + sv.Header.Set("Authorization", heartbeatBearer(testHubSecret, mktTestHive)) + sv.Header.Set(marketplaceHiveIDHeader, mktTestHive) + rec = mktDo(s, sv) + if rec.Code != 200 { + t.Fatalf("spoke verbose: %d %s", rec.Code, rec.Body.String()) + } + var verbose struct { + Failing []marketplaceFailingRepo `json:"failing"` + Pending []string `json:"pending"` + Pend int `json:"pendingRepos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &verbose); err != nil { + t.Fatal(err) + } + if len(verbose.Failing) != 1 || verbose.Failing[0].Repo != "broken/repo" || !strings.Contains(verbose.Failing[0].Error, "404") { + t.Fatalf("failing: %+v", verbose.Failing) + } + if verbose.Pending == nil || len(verbose.Pending) != verbose.Pend || verbose.Pend != 0 { + t.Fatalf("pending: %+v / %d", verbose.Pending, verbose.Pend) + } + if rec = mktDo(s, admin(http.MethodGet, "/api/marketplace/status?verbose=1")); rec.Code != 200 || !strings.Contains(rec.Body.String(), `"failing"`) { + t.Fatalf("admin verbose: %d %s", rec.Code, rec.Body.String()) + } +} + func TestMarketplaceReposMergesExtra(t *testing.T) { t.Setenv(hubMarketplaceExtraReposEnv, " Foo/Bar ,foo/bar, nope ,") repos := marketplaceRepos() @@ -645,26 +934,49 @@ func TestMarketplaceHonourRateLimit(t *testing.T) { h := http.Header{} h.Set("X-RateLimit-Remaining", "1000") - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != 0 { t.Fatal("plenty remaining must not sleep") } h.Set("X-RateLimit-Remaining", "3") h.Set("X-RateLimit-Reset", "garbage") - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != 0 { t.Fatal("unparseable reset must not sleep") } h.Set("X-RateLimit-Reset", "1") - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != 0 { t.Fatal("past reset must not sleep") } h.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(2*time.Hour).Unix(), 10)) - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != marketplaceRateMaxWait { t.Fatalf("far reset should cap at max wait, slept %v", slept) } + + // Secondary rate limit: 403/429 + Retry-After sleeps that long even + // with budget remaining; capped; a 403 without Retry-After does not. + slept = 0 + sec := http.Header{} + sec.Set("X-RateLimit-Remaining", "4000") + sec.Set("Retry-After", "7") + marketplaceHonourRateLimit(context.Background(), 403, sec) + if slept != 7*time.Second { + t.Fatalf("403 Retry-After: slept %v", slept) + } + slept = 0 + sec.Set("Retry-After", "99999") + marketplaceHonourRateLimit(context.Background(), 429, sec) + if slept != marketplaceRateMaxWait { + t.Fatalf("429 Retry-After cap: slept %v", slept) + } + slept = 0 + sec.Del("Retry-After") + marketplaceHonourRateLimit(context.Background(), 403, sec) + if slept != 0 { + t.Fatalf("403 without Retry-After and plenty remaining: slept %v", slept) + } } func TestMarketplaceStartRefreshDisabledAndSeedGate(t *testing.T) { @@ -680,8 +992,8 @@ func TestMarketplaceStartRefreshDisabledAndSeedGate(t *testing.T) { t.Setenv(hubMarketplaceDisableEnv, "") t.Setenv("GITHUB_TOKEN", "") t.Setenv("HIVE_HUB_GITHUB_TOKEN", "") - if s.marketplaceShouldSeed() { - t.Error("no token → no seed") + if due := s.marketplaceDueRepos(); due != nil { + t.Errorf("no token → nothing due, got %d", len(due)) } // Empty schedule: loop parks on ctx and exits on cancel. @@ -703,6 +1015,12 @@ func mktTarball(t *testing.T, files map[string]string) []byte { var buf bytes.Buffer gz := gzip.NewWriter(&buf) tw := tar.NewWriter(gz) + // GitHub tarballs (git archive) lead with a pax_global_header; the + // production bug behind #17620's one-entry catalogue was treating it as + // a second top-level directory. + if err := tw.WriteHeader(&tar.Header{Name: "pax_global_header", Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": "abc"}}); err != nil { + t.Fatal(err) + } for name, body := range files { if err := tw.WriteHeader(&tar.Header{Name: "repo-abc/" + name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { t.Fatal(err) @@ -778,8 +1096,8 @@ func TestMarketplaceGitHubFetcher(t *testing.T) { t.Error("branch lookup failure should fail") } - // With a token and an empty catalogue the seed gate opens. - if !s.marketplaceShouldSeed() { - t.Error("token + empty catalogue → should seed") + // With a token and no records every corpus repo is due. + if due := s.marketplaceDueRepos(); len(due) != len(marketplaceRepos()) { + t.Errorf("token + empty catalogue → all due, got %d", len(due)) } } From 8c665270a2a7371463fdfbf88402ba2f7bd9f22a Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:43:40 +0000 Subject: [PATCH 014/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20bound=20consecuti?= =?UTF-8?q?ve=20stale-snapshot=20drops=20in=20the=20status=20builder=20(#1?= =?UTF-8?q?7744)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: bound consecutive stale-snapshot drops in the status builder Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: update staleDropStarvationThreshold comment for bounded publish The constant's doc comment still described the pre-PR contract (drop logged at Warn). It now marks the point where the build is published anyway to bound staleness. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: resync api-reference citations after server.go route shift Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...fixed-status-snapshot-bounded-staleness.md | 1 + src/docs/api-reference/auth.md | 6 +-- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++++----- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/dashboard/server.go | 44 +++++++++++-------- src/pkg/dashboard/status_stale_drop_test.go | 31 ++++++++++--- 7 files changed, 67 insertions(+), 41 deletions(-) create mode 100644 changelog.d/fixed-status-snapshot-bounded-staleness.md diff --git a/changelog.d/fixed-status-snapshot-bounded-staleness.md b/changelog.d/fixed-status-snapshot-bounded-staleness.md new file mode 100644 index 0000000000..30daf850a4 --- /dev/null +++ b/changelog.d/fixed-status-snapshot-bounded-staleness.md @@ -0,0 +1 @@ +- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 36be77b0ee..08dba0af9e 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1681` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1682` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1683` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1683` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1684` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1685` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 1bdc09aed9..c8255286a9 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1688` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1690` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index c59b00248b..e41ee91d74 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1661` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1662` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1663` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1669` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1674` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1675` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1676` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1677` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1680` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1663` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1664` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1665` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1671` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1676` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1677` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1680` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1682` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 8dae7348f7..b8e71aaf96 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1751` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1753` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 233bebed05..7a72a42eb2 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -97,8 +97,10 @@ func isOwnerRole(role string) bool { // A WRONG proof header is ALWAYS rejected regardless of this setting. var proxyProofRequired = true -// staleDropStarvationThreshold is the number of consecutive stale-snapshot -// drops (no publish between) at which the drop is logged at Warn. +// staleDropStarvationThreshold bounds consecutive stale-snapshot drops (no +// publish between): the build that would be the Nth consecutive drop is +// published anyway, logged at Warn, so sustained mutation churn cannot +// starve consumers. The counter resets on every publish. const staleDropStarvationThreshold = 5 type Server struct { @@ -2797,24 +2799,30 @@ func (s *Server) UpdateStatusIfFresh(status *StatusPayload, buildEpoch uint64) b s.statusMu.Lock() if buildEpoch < s.statusMutationEpoch { - // A mutation landed after this snapshot's build began: its data may - // predate the mutation. Drop it — the mutation's own refresh rebuild - // (which began after the epoch bump) publishes the fresh state. curEpoch := s.statusMutationEpoch - s.statusStaleDrops++ - drops := s.statusStaleDrops - s.statusConsecutiveStaleDrops++ - consecutive := s.statusConsecutiveStaleDrops - s.statusMu.Unlock() - // A single drop is normal optimistic-concurrency self-healing; only a - // run of drops with no publish in between is an anomaly. - level := slog.LevelInfo - if consecutive >= staleDropStarvationThreshold { - level = slog.LevelWarn + if s.statusConsecutiveStaleDrops+1 >= staleDropStarvationThreshold { + // Bounded staleness: under sustained mutation churn every build + // loses the race, so publish this one rather than starve + // consumers. The counter resets below with the publish. + drops := s.statusStaleDrops + consecutive := s.statusConsecutiveStaleDrops + s.logger.Warn("publishing possibly-stale status snapshot after consecutive stale drops", + "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) + } else { + // A mutation landed after this snapshot's build began: its data + // may predate the mutation. Drop it — the mutation's own refresh + // rebuild (which began after the epoch bump) publishes the fresh + // state. + s.statusStaleDrops++ + drops := s.statusStaleDrops + s.statusConsecutiveStaleDrops++ + consecutive := s.statusConsecutiveStaleDrops + s.statusMu.Unlock() + // A drop is a normal optimistic-concurrency race. + s.logger.Info("dropping stale status snapshot built before a mutation", + "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) + return false } - s.logger.Log(context.Background(), level, "dropping stale status snapshot built before a mutation", - "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) - return false } // Attach the advisory ACMM level-up recommendation (#5225) from the SAME diff --git a/src/pkg/dashboard/status_stale_drop_test.go b/src/pkg/dashboard/status_stale_drop_test.go index 73c7abecb9..3ef712a878 100644 --- a/src/pkg/dashboard/status_stale_drop_test.go +++ b/src/pkg/dashboard/status_stale_drop_test.go @@ -97,21 +97,38 @@ func TestUpdateStatusIfFresh_StaleDropCounterAccumulates(t *testing.T) { } } -func TestUpdateStatusIfFresh_StarvationWarnsAfterThreshold(t *testing.T) { +func TestUpdateStatusIfFresh_StarvationPublishesAfterThreshold(t *testing.T) { var logBuf bytes.Buffer srv := NewServer(0, slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelInfo}))) for i := 0; i < staleDropStarvationThreshold-1; i++ { epoch := srv.BeginStatusSnapshot() srv.noteStatusMutation() - srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) + if srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) { + t.Fatalf("drop %d below the bound must not publish", i+1) + } } - if staleDropLoggedAt(logBuf.String(), "WARN") { - t.Fatalf("below the threshold must not WARN, got:\n%s", logBuf.String()) + if staleDropLoggedAt(logBuf.String(), "WARN") || strings.Contains(logBuf.String(), "level=WARN msg=\"publishing possibly-stale") { + t.Fatalf("below the bound must not WARN, got:\n%s", logBuf.String()) } + seq := srv.currentStatusSeq() epoch := srv.BeginStatusSnapshot() srv.noteStatusMutation() - srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) - if !staleDropLoggedAt(logBuf.String(), "WARN") { - t.Fatalf("threshold-th consecutive drop must WARN, got:\n%s", logBuf.String()) + if !srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) { + t.Fatalf("the bound-th stale build must publish, got:\n%s", logBuf.String()) + } + if got := srv.currentStatusSeq(); got != seq+1 { + t.Fatalf("statusSeq = %d, want %d", got, seq+1) + } + if n := strings.Count(logBuf.String(), "level=WARN msg=\"publishing possibly-stale"); n != 1 { + t.Fatalf("want exactly one bounded-staleness WARN, got %d:\n%s", n, logBuf.String()) + } + if staleDropLoggedAt(logBuf.String(), "WARN") { + t.Fatalf("per-drop line must never be WARN, got:\n%s", logBuf.String()) + } + srv.statusMu.RLock() + consecutive := srv.statusConsecutiveStaleDrops + srv.statusMu.RUnlock() + if consecutive != 0 { + t.Fatalf("counter must reset after the forced publish, got %d", consecutive) } } From 2939b0ebe31321e7df5c59b112cd885cac4946c1 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:44:02 +0000 Subject: [PATCH 015/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20log=20Prow=20ok-t?= =?UTF-8?q?o-test=20rejection=20at=20WARN=20only=20when=20first=20noted=20?= =?UTF-8?q?(#17754)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...fixed-prow-oktotest-rejection-warn-once.md | 1 + .../automerge_contributor_prow_oktotest.go | 11 +++++--- ...utomerge_contributor_prow_oktotest_test.go | 27 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-prow-oktotest-rejection-warn-once.md diff --git a/changelog.d/fixed-prow-oktotest-rejection-warn-once.md b/changelog.d/fixed-prow-oktotest-rejection-warn-once.md new file mode 100644 index 0000000000..6e33a4e8bb --- /dev/null +++ b/changelog.d/fixed-prow-oktotest-rejection-warn-once.md @@ -0,0 +1 @@ +- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go index f1ebb2f69c..04e9252929 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go @@ -68,11 +68,14 @@ func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, } if state.rejected { c.markWorkflowApprovalUntrusted(owner, repo, number) - if !state.rejectionNoted { - c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowPresubmitsRejectedNote(strings.TrimSpace(c.transport.AppBotLogin()))) + args := []any{"repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw} + const msg = "contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it" + if state.rejectionNoted { + c.debug(msg, args...) + return } - c.warn("contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it", - "repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw) + c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowPresubmitsRejectedNote(strings.TrimSpace(c.transport.AppBotLogin()))) + c.warn(msg, args...) return } if state.postedForHead || !(hgithub.ProwManagedLabels(labels) || state.prowManaged) { diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go index b884c6cfd8..2a985bd06a 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go @@ -7,7 +7,9 @@ package automerge // that shape, never otherwise, and stops for good once Prow rejects the App. import ( + "bytes" "context" + "log/slog" "strings" "testing" "time" @@ -181,3 +183,28 @@ func TestContributorLaneRelayEvaluationDoesNotPostProwOkToTest(t *testing.T) { t.Fatalf("comments = %d %q, want none from a relay evaluation", fx.comments, fx.allComments) } } + +// A rejection the PR thread already records is a handled state: a restarted +// hive must not log it at WARN again. +func TestContributorLaneProwRejectionAlreadyNotedLogsBelowWarn(t *testing.T) { + fx := prowOkToTestFixture() + fx.prComments = []map[string]any{ + appComment(2, workflowApprovalCommandBody("oldhead")), + prowComment(3, "@"+testHiveAppBotLogin+": Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message."), + } + api := newContributorLaneAPI(t, &fx) + defer api.Close() + policy := defaultContributorLanePolicy() + clock := &contributorLaneClock{at: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)} + var logs bytes.Buffer + for run := 0; run < 2; run++ { + engine := newContributorLaneEngine(api.URL, policy, clock, Options{ContributorSoak: soakedTracker(clock)}) + engine.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, _, err := engine.trySweepContributorLanePR(context.Background(), "widget", "acme", "widget", 7, policy); err != nil { + t.Fatalf("run %d err = %v", run, err) + } + } + if got := strings.Count(logs.String(), "ok-to-test not posted again"); got != 1 { + t.Fatalf("WARN lines = %d in %q, want 1 (only the run that left the note)", got, logs.String()) + } +} From faec2aa2dc24da3a4a60820b8e4c69080ba2fb42 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 15:51:44 -0700 Subject: [PATCH 016/223] =?UTF-8?q?=E2=9C=A8=20crux:=20Substrate=20preflig?= =?UTF-8?q?ht=20prover=20(operator,=20CRDs,=20gVisor=20RuntimeClass,=20fea?= =?UTF-8?q?ture=20gates)=20(#17766)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: Substrate preflight prover (operator, CRDs, gVisor RuntimeClass, feature gates) Extends the sandbox.runtime: substrate preflight with the cluster facts behind a failing kick: operator CRDs and workloads, ready WorkerPools and their sandbox classes, gVisor (SandboxConfig or runsc RuntimeClass) and KVM availability, node kernels, the certificates.k8s.io APIs behind the PodCertificateRequest and ClusterTrustBundle feature gates, and an mTLS CONNECT probe of atenet-router. Skipped outside a cluster; one warning when RBAC is missing, with a manifest that grants it. Fixes #17698 Signed-off-by: Andrew Anderson * fix(crux/sandbox): egress policy atespace + in-place workspace clear found by #17697 kick The first real kick through Substrate v1.0.0 (kind, gVisor) surfaced two defects in the launcher path: CreateActorEgressPolicy was rejected with 'egress_policy.metadata.atespace: Required value', and the bridge's PUT /workspace failed with EBUSY because os.RemoveAll targeted the durableDir mount point itself. Set the atespace on the singleton policy and clear the workspace directory's contents in place. Signed-off-by: Andrew Anderson * test(crux/sandbox): run the kube/router preflight halves only in a Pod wired for Substrate CI runs in Pods, so /var/run/secrets/kubernetes.io/serviceaccount/token exists there and the kube/router halves of the prover ran during the Control-API-only tests (crux/sandbox and crux/agent). The prover now requires both the Kubernetes token and hive's projected Substrate token (opts.token_file, /var/run/secrets/ate/token by default) before probing the cluster; the Kubernetes token path is a package variable the sandbox test package pins to a nonexistent file, with a helper that flips both markers on for the tests covering the in-cluster branches. Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../added-substrate-preflight-prover.md | 1 + src/crux/sandbox/substrate.go | 3 + src/crux/sandbox/substrate_preflight.go | 23 +- .../sandbox/substrate_preflight_cluster.go | 534 ++++++++++++++++++ .../substrate_preflight_cluster_test.go | 513 +++++++++++++++++ src/crux/sandbox/substratebridge/bridge.go | 21 +- .../sandbox/substratebridge/bridge_test.go | 47 ++ .../k8s/sandbox-substrate-preflight-rbac.yaml | 72 +++ 8 files changed, 1209 insertions(+), 5 deletions(-) create mode 100644 changelog.d/added-substrate-preflight-prover.md create mode 100644 src/crux/sandbox/substrate_preflight_cluster.go create mode 100644 src/crux/sandbox/substrate_preflight_cluster_test.go create mode 100644 src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml diff --git a/changelog.d/added-substrate-preflight-prover.md b/changelog.d/added-substrate-preflight-prover.md new file mode 100644 index 0000000000..dd9d5bd5d9 --- /dev/null +++ b/changelog.d/added-substrate-preflight-prover.md @@ -0,0 +1 @@ +- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go index 1ed56c0ee9..9ad2d81f2d 100644 --- a/src/crux/sandbox/substrate.go +++ b/src/crux/sandbox/substrate.go @@ -303,6 +303,9 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e if spec.NetworkNone { policy.Rules = nil } + // The Control API requires the singleton policy's atespace to match the + // actor's (egress_policy.metadata.atespace is a required field). + policy.Metadata.Atespace = ref.Atespace if err := control.CreateActorEgressPolicy(ctx, ref, policy); err != nil { cleanupActor() return res, fmt.Errorf("substrate: setting egress policy on actor %s: %w", name, err) diff --git a/src/crux/sandbox/substrate_preflight.go b/src/crux/sandbox/substrate_preflight.go index 94964cc5e9..e166591dfa 100644 --- a/src/crux/sandbox/substrate_preflight.go +++ b/src/crux/sandbox/substrate_preflight.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "fmt" + "net" "sort" "strings" "sync" @@ -12,10 +13,11 @@ import ( ) // Substrate cluster-prerequisite preflight (ADR-0026 phase 2; the sibling -// of the OpenShell preflight in openshell_preflight.go). Hive cannot see the -// node kernel or the WorkerPool CRDs from its pod without cluster RBAC, but -// the Control API answers whether it is up and which workers — and so which -// sandbox classes — exist; that is what decides whether a kick can land. +// of the OpenShell preflight in openshell_preflight.go). The Control API +// answers whether it is up and which workers — and so which sandbox classes +// — exist; that is what decides whether a kick can land. The cluster and +// router halves in substrate_preflight_cluster.go (#17698) add the +// Kubernetes facts behind that answer when hive runs in the cluster. const substratePreflightTTL = 5 * time.Minute @@ -32,6 +34,10 @@ type SubstratePreflighter struct { newClient func(SubstrateOptions) (substrateClient, error) now func() time.Time + // kube and routerDial let tests point the cluster and router halves at + // fakes; nil means the in-cluster API and a plain TCP dial. + kube *substrateKube + routerDial func(ctx context.Context, network, addr string) (net.Conn, error) } // DefaultSubstratePreflighter backs the warnings AgentSandboxGateWarnings emits. @@ -58,7 +64,16 @@ func (p *SubstratePreflighter) Check(ctx context.Context, opts SubstrateOptions) return append([]string(nil), warnings...) } +// probe runs the three halves — Control API, cluster facts, router — and +// concatenates their warnings; a down control plane does not hide the +// cluster facts that explain it. func (p *SubstratePreflighter) probe(ctx context.Context, opts SubstrateOptions) []string { + out := p.probeControl(ctx, opts) + out = append(out, p.probeCluster(ctx, opts)...) + return append(out, p.probeRouter(ctx, opts)...) +} + +func (p *SubstratePreflighter) probeControl(ctx context.Context, opts SubstrateOptions) []string { ep := opts.endpoint() ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) defer cancel() diff --git a/src/crux/sandbox/substrate_preflight_cluster.go b/src/crux/sandbox/substrate_preflight_cluster.go new file mode 100644 index 0000000000..bf9de8e81a --- /dev/null +++ b/src/crux/sandbox/substrate_preflight_cluster.go @@ -0,0 +1,534 @@ +package sandbox + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Cluster-level half of the Substrate preflight (hivecommons/hive#17698). +// The Control API half in substrate_preflight.go answers "is the control +// plane up and does it have workers"; this half reads the Kubernetes facts +// that explain *why* it is not, or why a kick would fail even though it is: +// the operator's CRDs and Deployments, a ready WorkerPool for the class the +// operator selected, gVisor (a SandboxConfig of class gvisor, or a runsc +// RuntimeClass) or KVM for microVMs, the certificates.k8s.io APIs behind the +// PodCertificateRequest and ClusterTrustBundle feature gates Substrate's +// SPIFFE identities need, the node kernels, and whether atenet-router +// accepts hive's client certificate. Everything is read-only, bounded by one +// request timeout per call, and skipped entirely when hive is not running in +// a cluster (no projected ServiceAccount token), where the Control API probe +// is all there is to check. + +const ( + // SubstrateDefaultOperatorNamespace is where ate-setup installs the + // control plane when the operator does not rename it. + SubstrateDefaultOperatorNamespace = "ate-system" + + substrateKubeAPIServer = "https://kubernetes.default.svc" + substrateKubeCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + substrateKubeMaxBody = 8 << 20 + substrateKubeNodeLimit = 500 + + substrateClassGVisor = "gvisor" + substrateClassMicroVM = "microvm" + substrateGVisorHandler = "runsc" + // substrateKVMResource is the extended resource atelet's device plugin + // advertises on nodes with /dev/kvm; the microVM class schedules on it. + substrateKVMResource = "ate.dev/kvm" + // substrateGVisorMinKernel is gVisor's documented host kernel floor. + substrateGVisorMinKernel = "4.14.77" + + substrateAteletDaemonSet = "atelet" + substrateCertsGroupPath = "/apis/certificates.k8s.io/" + substrateResourcePCR = "podcertificaterequests" + substrateResourceCTB = "clustertrustbundles" +) + +// substrateCRDs are the cluster-scoped CRDs ate-setup installs; Actors and +// ActorTemplates are Control API resources, not CRDs, so they are not here. +var substrateCRDs = []string{"workerpools.ate.dev", "sandboxconfigs.ate.dev"} + +// substrateOperatorDeployments must each have a ready replica for a kick to +// create an actor, schedule it, reach its bridge, and egress to GitHub. +var substrateOperatorDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress"} + +// substrateCertsVersions are the certificates.k8s.io versions that may serve +// PodCertificateRequest and ClusterTrustBundle (beta in 1.34, GA later). +var substrateCertsVersions = []string{"v1", "v1beta1"} + +// substrateKubeTokenPath is the projected Kubernetes ServiceAccount token +// the kube client authenticates with. A variable so tests pin the +// environment instead of inheriting the runner's (CI itself runs in Pods). +var substrateKubeTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +// substrateInCluster reports whether hive runs in a Pod wired for Substrate: +// both the Kubernetes ServiceAccount token and hive's projected Substrate +// token exist. The kube and router halves of the preflight only run then; +// a laptop, or a CI Pod without the Substrate projection, gets the +// Control-API probe alone. +func substrateInCluster(opts SubstrateOptions) bool { + if _, err := os.Stat(substrateKubeTokenPath); err != nil { + return false + } + _, err := os.Stat(opts.tokenFile()) + return err == nil +} + +// substrateKube is the minimal read-only in-cluster API client the prover +// needs; kubejob has its own because importing it here would be a cycle. +type substrateKube struct { + server string + tokenPath string + caPath string + client *http.Client +} + +// errSubstrateKubeForbidden marks a 403: the probe stops after one warning +// rather than repeating the same RBAC gap per resource. +var errSubstrateKubeForbidden = errors.New("forbidden") + +// errSubstrateKubeNotFound marks a 404. +var errSubstrateKubeNotFound = errors.New("not found") + +func (k *substrateKube) get(ctx context.Context, path string) ([]byte, error) { + client, err := k.httpClient() + if err != nil { + return nil, err + } + reqCtx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, k.server+path, nil) + if err != nil { + return nil, err + } + if raw, err := os.ReadFile(k.tokenPath); err == nil { + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(raw))) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, substrateKubeMaxBody)) + if err != nil { + return nil, err + } + switch { + case resp.StatusCode == http.StatusForbidden: + return nil, errSubstrateKubeForbidden + case resp.StatusCode == http.StatusNotFound: + return nil, errSubstrateKubeNotFound + case resp.StatusCode/100 != 2: + return nil, fmt.Errorf("GET %s: HTTP %d", path, resp.StatusCode) + } + return body, nil +} + +func (k *substrateKube) httpClient() (*http.Client, error) { + if k.client != nil { + return k.client, nil + } + pem, err := os.ReadFile(k.caPath) + if err != nil { + return nil, fmt.Errorf("reading in-cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("in-cluster CA bundle contained no certificates") + } + k.client = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + return k.client, nil +} + +// substrateOperatorNamespace derives the operator namespace from the Control +// API endpoint (`api..svc[:port]`, with or without a scheme); any other +// shape means the default install. +func substrateOperatorNamespace(endpoint string) string { + addr, _, err := substrateAddress(endpoint) + if err != nil { + return SubstrateDefaultOperatorNamespace + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + parts := strings.Split(host, ".") + if len(parts) >= 3 && parts[0] == "api" && parts[2] == "svc" && parts[1] != "" { + return parts[1] + } + return SubstrateDefaultOperatorNamespace +} + +// Minimal projections of the API objects the prover reads. + +type substrateKubeList struct { + Items []json.RawMessage `json:"items"` +} + +type substrateKubeWorkload struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + SandboxClass string `json:"sandboxClass"` + SandboxClasses []struct { + Name string `json:"name"` + } `json:"sandboxClasses"` + } `json:"spec"` + Status struct { + ReadyReplicas int `json:"readyReplicas"` + NumberReady int `json:"numberReady"` + } `json:"status"` + Handler string `json:"handler"` +} + +type substrateKubeNode struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Allocatable map[string]string `json:"allocatable"` + NodeInfo struct { + KernelVersion string `json:"kernelVersion"` + } `json:"nodeInfo"` + } `json:"status"` +} + +type substrateKubeResourceList struct { + Resources []struct { + Name string `json:"name"` + } `json:"resources"` +} + +// clusterProbe holds one probe's state so the step functions stay short. +type substrateClusterProbe struct { + kube *substrateKube + ns string + sel map[string]string + out []string + // classes are the sandbox classes the ready WorkerPools offer; gvisor and + // kvm record whether the cluster can run each class. + classes map[string]bool + gvisor bool + kvm bool +} + +// substrateKubeItems GETs a collection and decodes each item into T. +func substrateKubeItems[T any](ctx context.Context, k *substrateKube, path string) ([]T, error) { + body, err := k.get(ctx, path) + if err != nil { + return nil, err + } + var list substrateKubeList + if err := json.Unmarshal(body, &list); err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + items := make([]T, 0, len(list.Items)) + for _, raw := range list.Items { + var item T + if err := json.Unmarshal(raw, &item); err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + items = append(items, item) + } + return items, nil +} + +// warnf appends one warning. +func (c *substrateClusterProbe) warnf(format string, args ...any) { + c.out = append(c.out, fmt.Sprintf(format, args...)) +} + +// probeCluster runs the Kubernetes checks and returns their warnings. A 403 +// anywhere ends the probe with one warning naming the RBAC the prover needs. +func (p *SubstratePreflighter) probeCluster(ctx context.Context, opts SubstrateOptions) []string { + kube := p.kube + if kube == nil { + if !substrateInCluster(opts) { + return nil + } + kube = &substrateKube{server: substrateKubeAPIServer, tokenPath: substrateKubeTokenPath, caPath: substrateKubeCAPath} + } + c := &substrateClusterProbe{kube: kube, ns: substrateOperatorNamespace(opts.endpoint()), sel: opts.WorkerSelector, classes: map[string]bool{}} + steps := []func(context.Context) error{c.crds, c.operator, c.pools, c.sandboxClasses, c.featureGates, c.nodes} + for _, step := range steps { + if err := step(ctx); err != nil { + if errors.Is(err, errSubstrateCRDsMissing) { + return c.out + } + if errors.Is(err, errSubstrateKubeForbidden) { + return append(c.out, "hive's ServiceAccount may not read the Substrate cluster facts (403); the operator, WorkerPool, gVisor and feature-gate checks were skipped — apply deploy/k8s/sandbox-substrate-preflight-rbac.yaml") + } + c.warnf("substrate cluster facts could not be read (%v); the operator, WorkerPool, gVisor and feature-gate checks were skipped", err) + return c.out + } + } + return c.out +} + +// crds checks the operator's CRDs are installed; missing ones end the probe +// early, because every later check would just restate "not installed". +func (c *substrateClusterProbe) crds(ctx context.Context) error { + missing := 0 + for _, name := range substrateCRDs { + _, err := c.kube.get(ctx, "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/"+name) + switch { + case err == nil: + case errors.Is(err, errSubstrateKubeNotFound): + missing++ + c.warnf("substrate CRD %s is not installed; the Substrate operator is absent and every kick on the substrate runtime will fail — run `ate-setup deploy ate-system` (see deploy/substrate.md)", name) + default: + return err + } + } + if missing > 0 { + return errSubstrateCRDsMissing + } + return nil +} + +// errSubstrateCRDsMissing stops the probe after the CRD warnings. +var errSubstrateCRDsMissing = errors.New("crds missing") + +// operator checks each control-plane Deployment and the atelet DaemonSet +// have a ready replica in the operator namespace. +func (c *substrateClusterProbe) operator(ctx context.Context) error { + deps, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/apps/v1/namespaces/"+c.ns+"/deployments") + if err != nil { + return err + } + ready := map[string]int{} + for _, d := range deps { + ready[d.Metadata.Name] = d.Status.ReadyReplicas + } + for _, name := range substrateOperatorDeployments { + n, ok := ready[name] + switch { + case !ok: + c.warnf("substrate operator deployment %s/%s is not installed; run `ate-setup deploy ate-system` (see deploy/substrate.md)", c.ns, name) + case n == 0: + c.warnf("substrate operator deployment %s/%s has no ready replica; kicks on the substrate runtime will fail until it is up (kubectl -n %s describe deploy %s)", c.ns, name, c.ns, name) + } + } + body, err := c.kube.get(ctx, "/apis/apps/v1/namespaces/"+c.ns+"/daemonsets/"+substrateAteletDaemonSet) + switch { + case errors.Is(err, errSubstrateKubeNotFound): + c.warnf("substrate node agent daemonset %s/%s is not installed; no node can host a worker — run `ate-setup deploy ate-system`", c.ns, substrateAteletDaemonSet) + return nil + case err != nil: + return err + } + var ds substrateKubeWorkload + if err := json.Unmarshal(body, &ds); err != nil { + return fmt.Errorf("daemonset %s: %w", substrateAteletDaemonSet, err) + } + if ds.Status.NumberReady == 0 { + c.warnf("substrate node agent daemonset %s/%s has no ready pod; workers cannot start (kubectl -n %s describe ds %s)", c.ns, substrateAteletDaemonSet, c.ns, substrateAteletDaemonSet) + } + return nil +} + +// pools checks at least one WorkerPool has a ready replica and records the +// sandbox classes those pools offer. +func (c *substrateClusterProbe) pools(ctx context.Context) error { + pools, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/ate.dev/v1/workerpools") + if err != nil { + return err + } + ready := 0 + for _, p := range pools { + if p.Status.ReadyReplicas == 0 { + continue + } + ready++ + for _, sc := range p.Spec.SandboxClasses { + c.classes[strings.ToLower(sc.Name)] = true + } + } + if len(pools) == 0 { + c.warnf("no substrate WorkerPool exists; actors have nowhere to run — create one (see deploy/substrate.md)") + } else if ready == 0 { + c.warnf("none of the %d substrate WorkerPool(s) has a ready replica; actors will never be scheduled (kubectl get workerpools -A)", len(pools)) + } + return nil +} + +// sandboxClasses checks the cluster can actually run the classes the pools +// offer: gVisor via a SandboxConfig of class gvisor (atelet fetches runsc +// itself) or a runsc RuntimeClass; microVMs via atelet's ate.dev/kvm +// resource on at least one node (checked in nodes). +func (c *substrateClusterProbe) sandboxClasses(ctx context.Context) error { + cfgs, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/ate.dev/v1/sandboxconfigs") + if err != nil { + return err + } + for _, cfg := range cfgs { + if strings.EqualFold(cfg.Spec.SandboxClass, substrateClassGVisor) { + c.gvisor = true + } + } + rcs, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/node.k8s.io/v1/runtimeclasses") + if err != nil && !errors.Is(err, errSubstrateKubeNotFound) { + return err + } + for _, rc := range rcs { + if rc.Handler == substrateGVisorHandler { + c.gvisor = true + } + } + wantGVisor := c.classes[substrateClassGVisor] || len(c.classes) == 0 + if wantGVisor && !c.gvisor { + c.warnf("cluster has no gVisor: no SandboxConfig with sandboxClass %s and no RuntimeClass with handler %s; Substrate refuses plain containers, so a gvisor WorkerPool cannot start workers — `ate-setup deploy ate-system` installs the gvisor-default SandboxConfig", substrateClassGVisor, substrateGVisorHandler) + } + return nil +} + +// featureGates checks the certificates.k8s.io APIs that only exist when the +// PodCertificateRequest and ClusterTrustBundle feature gates are on; without +// them atelet, atenet-router and hive's own router client get no SPIFFE +// certificates and every CONNECT to an actor fails. +func (c *substrateClusterProbe) featureGates(ctx context.Context) error { + served := map[string]bool{} + for _, v := range substrateCertsVersions { + body, err := c.kube.get(ctx, substrateCertsGroupPath+v) + if errors.Is(err, errSubstrateKubeNotFound) { + continue + } + if err != nil { + return err + } + var rl substrateKubeResourceList + if err := json.Unmarshal(body, &rl); err != nil { + return fmt.Errorf("certificates.k8s.io/%s: %w", v, err) + } + for _, r := range rl.Resources { + served[strings.ToLower(r.Name)] = true + } + } + if !served[substrateResourcePCR] { + c.warnf("the PodCertificateRequest API is not served; Substrate's SPIFFE identities (atelet, atenet-router, hive's router client) cannot be issued — enable the PodCertificateRequest feature gate and certificates.k8s.io/v1beta1 on the API server and kubelets (Kubernetes ≥ 1.34)") + } + if !served[substrateResourceCTB] { + c.warnf("the ClusterTrustBundle API is not served; the Control API and router CA bundles cannot be projected into hive — enable the ClusterTrustBundle feature gate and certificates.k8s.io/v1beta1 on the API server and kubelets (Kubernetes ≥ 1.34)") + } + return nil +} + +// nodes surveys the node kernels against gVisor's floor and records whether +// any node advertises KVM for the microVM class. +func (c *substrateClusterProbe) nodes(ctx context.Context) error { + nodes, err := substrateKubeItems[substrateKubeNode](ctx, c.kube, "/api/v1/nodes?limit="+strconv.Itoa(substrateKubeNodeLimit)) + if err != nil { + return err + } + var old []string + for _, n := range nodes { + if _, ok := n.Status.Allocatable[substrateKVMResource]; ok { + c.kvm = true + } + if kv := n.Status.NodeInfo.KernelVersion; kv != "" && substrateKernelLess(kv, substrateGVisorMinKernel) { + old = append(old, n.Metadata.Name+" ("+kv+")") + } + } + if len(old) > 0 { + sort.Strings(old) + c.warnf("%d node(s) run a kernel older than gVisor's %s floor and cannot host a gvisor worker: %s", len(old), substrateGVisorMinKernel, strings.Join(old, ", ")) + } + if c.classes[substrateClassMicroVM] && !c.kvm { + c.warnf("a WorkerPool offers the %s class but no node advertises %s; microVM workers need /dev/kvm (KVM-capable shapes) and atelet's device plugin", substrateClassMicroVM, substrateKVMResource) + } + return nil +} + +// substrateKernelLess reports whether kernel release a (e.g. 5.15.0-1 or +// 6.12.0-207.el9uek.aarch64) is older than the dotted floor b. A release the +// prover cannot parse is never reported as old. +func substrateKernelLess(a, b string) bool { + pa, oka := substrateKernelParts(a) + pb, okb := substrateKernelParts(b) + if !oka || !okb { + return false + } + for i := range pa { + if pa[i] != pb[i] { + return pa[i] < pb[i] + } + } + return false +} + +func substrateKernelParts(s string) ([3]int, bool) { + var out [3]int + s, _, _ = strings.Cut(s, "-") + parts := strings.Split(s, ".") + if len(parts) < 2 { + return out, false + } + for i := 0; i < 3 && i < len(parts); i++ { + n, err := strconv.Atoi(parts[i]) + if err != nil { + return out, false + } + out[i] = n + } + return out, true +} + +// substrateRouterProbeActor is the (nonexistent) actor the preflight names +// in its CONNECT: the router answers with an HTTP status once it has +// accepted hive's client certificate, and with a TLS alert when it has not. +const substrateRouterProbeActor = "hive-preflight" + +// probeRouter checks hive can reach atenet-router and that the router +// accepts its projected client bundle: the same mTLS CONNECT the kick's +// bridge tunnel makes, aimed at an actor that does not exist. Any HTTP +// answer means the certificate was accepted (TLS 1.3 reports a rejected +// client certificate only after the handshake, so the handshake alone is +// not proof). Skipped outside a cluster, where neither the bundle nor the +// router exist. +func (p *SubstratePreflighter) probeRouter(ctx context.Context, opts SubstrateOptions) []string { + if p.kube == nil && !substrateInCluster(opts) { + return nil + } + bridge, err := newRouterBridge(opts) + if err != nil { + return []string{fmt.Sprintf("atenet-router client credentials are not usable (%v); every kick on the substrate runtime will fail at the bridge tunnel", err)} + } + if p.routerDial != nil { + bridge.dial = p.routerDial + } + ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + ref := &substrateapi.ObjectRef{Atespace: opts.atespace(), Name: substrateRouterProbeActor} + conn, err := bridge.tunnel(ctx, ref) + if conn != nil { + _ = conn.Close() + } + switch { + case err == nil, strings.Contains(err.Error(), "router refused connect"): + return nil + case strings.Contains(err.Error(), "router tls handshake") || strings.Contains(err.Error(), "router connect"): + return []string{fmt.Sprintf("atenet-router %s rejected hive's client certificate (%v); check the podCertificate projection at %s and --ingress-allowed-spiffe-ids on the router", bridge.routerAddr, err, substrateRouterBundlePath(opts))} + } + return []string{fmt.Sprintf("atenet-router %s is unreachable (%v); every kick on the substrate runtime will fail at the bridge tunnel — check the atenet-router Service and NetworkPolicies", bridge.routerAddr, err)} +} + +func substrateRouterBundlePath(opts SubstrateOptions) string { + if opts.RouterClientBundle != "" { + return opts.RouterClientBundle + } + return SubstrateDefaultRouterClientBundle +} diff --git a/src/crux/sandbox/substrate_preflight_cluster_test.go b/src/crux/sandbox/substrate_preflight_cluster_test.go new file mode 100644 index 0000000000..0cefac58f0 --- /dev/null +++ b/src/crux/sandbox/substrate_preflight_cluster_test.go @@ -0,0 +1,513 @@ +package sandbox + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Cluster/router half of the Substrate preflight (#17698): a fake Kubernetes +// API serves the handful of read-only endpoints the prover walks, and a TLS +// listener plays atenet-router. + +// fakeKubeAPI is a path → JSON (or status code) table. +type fakeKubeAPI struct { + bodies map[string]any + codes map[string]int + calls []string +} + +func (f *fakeKubeAPI) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.calls = append(f.calls, r.URL.Path) + if code, ok := f.codes[r.URL.Path]; ok { + w.WriteHeader(code) + return + } + body, ok := f.bodies[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if raw, isRaw := body.(string); isRaw { + _, _ = w.Write([]byte(raw)) + return + } + _ = json.NewEncoder(w).Encode(body) + }) +} + +func kubeList(items ...any) map[string]any { return map[string]any{"items": items} } + +func kubeDeploy(name string, ready int) map[string]any { + return map[string]any{"metadata": map[string]any{"name": name}, "status": map[string]any{"readyReplicas": ready}} +} + +func kubePool(name string, ready int, classes ...string) map[string]any { + var sc []map[string]any + for _, c := range classes { + sc = append(sc, map[string]any{"name": c}) + } + return map[string]any{"metadata": map[string]any{"name": name}, "spec": map[string]any{"sandboxClasses": sc}, "status": map[string]any{"readyReplicas": ready}} +} + +func kubeNode(name, kernel string, kvm bool) map[string]any { + alloc := map[string]string{"cpu": "4"} + if kvm { + alloc[substrateKVMResource] = "1" + } + return map[string]any{"metadata": map[string]any{"name": name}, "status": map[string]any{"allocatable": alloc, "nodeInfo": map[string]any{"kernelVersion": kernel}}} +} + +func kubeResources(names ...string) map[string]any { + var rs []map[string]any + for _, n := range names { + rs = append(rs, map[string]any{"name": n}) + } + return map[string]any{"resources": rs} +} + +const ( + pathCRDs = "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + pathDeploys = "/apis/apps/v1/namespaces/ate-system/deployments" + pathAtelet = "/apis/apps/v1/namespaces/ate-system/daemonsets/atelet" + pathPools = "/apis/ate.dev/v1/workerpools" + pathSandboxes = "/apis/ate.dev/v1/sandboxconfigs" + pathRuntimes = "/apis/node.k8s.io/v1/runtimeclasses" + pathNodes = "/api/v1/nodes" +) + +// healthyKube is a cluster where every check passes. +func healthyKube() *fakeKubeAPI { + f := &fakeKubeAPI{bodies: map[string]any{}, codes: map[string]int{}} + for _, crd := range substrateCRDs { + f.bodies[pathCRDs+crd] = map[string]any{"metadata": map[string]any{"name": crd}} + } + var deps []any + for _, d := range substrateOperatorDeployments { + deps = append(deps, kubeDeploy(d, 1)) + } + f.bodies[pathDeploys] = kubeList(deps...) + f.bodies[pathAtelet] = map[string]any{"status": map[string]any{"numberReady": 3}} + f.bodies[pathPools] = kubeList(kubePool("gv", 2, "gvisor")) + f.bodies[pathSandboxes] = kubeList(map[string]any{"spec": map[string]any{"sandboxClass": "gvisor"}}) + f.bodies[pathRuntimes] = kubeList() + f.bodies[substrateCertsGroupPath+"v1beta1"] = kubeResources("podcertificaterequests", "clustertrustbundles") + f.bodies[pathNodes] = kubeList(kubeNode("n1", "6.12.0-207.el9uek.aarch64", false)) + return f +} + +// clusterPreflighter returns a preflighter whose Control API half is healthy +// and whose cluster half talks to the fake, with the router half stubbed to +// a refused dial unless the test overrides it. +func clusterPreflighter(t *testing.T, f *fakeKubeAPI) (*SubstratePreflighter, SubstrateOptions) { + t.Helper() + srv := httptest.NewServer(f.handler()) + t.Cleanup(srv.Close) + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + token := filepath.Join(dir, "token") + if err := os.WriteFile(token, []byte("sa-token"), 0o600); err != nil { + t.Fatal(err) + } + fake := &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil, "gvisor")}} + p := &SubstratePreflighter{ + newClient: fakeSubstrateFactory{fake: fake}.new, + kube: &substrateKube{server: srv.URL, tokenPath: token, caPath: ca, client: srv.Client()}, + routerDial: func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("dial stubbed") + }, + } + opts := SubstrateOptions{Endpoint: "api.ate-system.svc:443", Token: "t", RouterClientBundle: bundle, RouterCAFile: ca} + return p, opts +} + +func TestSubstratePreflightClusterHealthyOnlyWarnsAboutRouter(t *testing.T) { + f := healthyKube() + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + if len(got) != 1 || !strings.Contains(got[0], "atenet-router") || !strings.Contains(got[0], "dial stubbed") { + t.Fatalf("healthy cluster warnings = %q", got) + } + if !strings.Contains(strings.Join(f.calls, " "), pathNodes) { + t.Errorf("node survey not reached: %v", f.calls) + } +} + +func TestSubstratePreflightClusterMissingCRDsStopsEarly(t *testing.T) { + f := healthyKube() + delete(f.bodies, pathCRDs+"workerpools.ate.dev") + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + var crd int + for _, w := range got { + if strings.Contains(w, "CRD workerpools.ate.dev is not installed") { + crd++ + } + if strings.Contains(w, "deployment") || strings.Contains(w, "WorkerPool") { + t.Errorf("operator checks ran after missing CRDs: %q", w) + } + } + if crd != 1 { + t.Errorf("CRD warnings = %d in %q", crd, got) + } +} + +func TestSubstratePreflightClusterForbiddenIsOneWarning(t *testing.T) { + f := healthyKube() + f.codes[pathDeploys] = http.StatusForbidden + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + n := 0 + for _, w := range got { + if strings.Contains(w, "403") && strings.Contains(w, "sandbox-substrate-preflight-rbac.yaml") { + n++ + } + } + if n != 1 || len(got) != 2 { + t.Errorf("forbidden warnings = %q", got) + } +} + +func TestSubstratePreflightClusterReadErrorIsAWarning(t *testing.T) { + f := healthyKube() + f.codes[pathPools] = http.StatusInternalServerError + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + if len(got) != 2 || !strings.Contains(got[0], "could not be read") || !strings.Contains(got[0], "HTTP 500") { + t.Errorf("read error warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathPools] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "could not be read") { + t.Errorf("bad json warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathPools] = kubeList("not an object") + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "could not be read") { + t.Errorf("bad item warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathAtelet] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "daemonset atelet") { + t.Errorf("bad daemonset warnings = %q", got) + } + + f = healthyKube() + f.bodies[substrateCertsGroupPath+"v1beta1"] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "certificates.k8s.io/v1beta1") { + t.Errorf("bad discovery warnings = %q", got) + } +} + +func TestSubstratePreflightClusterOperatorAndPoolWarnings(t *testing.T) { + f := healthyKube() + f.bodies[pathDeploys] = kubeList(kubeDeploy("ate-api-server", 0), kubeDeploy("ate-controller", 1), kubeDeploy("atenet-router", 1)) + f.bodies[pathAtelet] = map[string]any{"status": map[string]any{"numberReady": 0}} + f.bodies[pathPools] = kubeList(kubePool("gv", 0, "gvisor")) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + for _, want := range []string{ + "deployment ate-system/ate-api-server has no ready replica", + "deployment ate-system/atenet-egress is not installed", + "daemonset ate-system/atelet has no ready pod", + "none of the 1 substrate WorkerPool(s) has a ready replica", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + if strings.Contains(got, "ate-controller") || strings.Contains(got, "atenet-router is") { + t.Errorf("healthy deployments warned:\n%s", got) + } + + f = healthyKube() + delete(f.bodies, pathAtelet) + f.bodies[pathPools] = kubeList() + p, opts = clusterPreflighter(t, f) + got = strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "daemonset ate-system/atelet is not installed") || !strings.Contains(got, "no substrate WorkerPool exists") { + t.Errorf("missing atelet/pool warnings:\n%s", got) + } +} + +func TestSubstratePreflightClusterSandboxClassWarnings(t *testing.T) { + // No gvisor SandboxConfig and no runsc RuntimeClass → warning. + f := healthyKube() + f.bodies[pathSandboxes] = kubeList() + delete(f.bodies, pathRuntimes) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "cluster has no gVisor") { + t.Errorf("missing gVisor warning:\n%s", got) + } + + // A runsc RuntimeClass alone satisfies gVisor. + f = healthyKube() + f.bodies[pathSandboxes] = kubeList() + f.bodies[pathRuntimes] = kubeList(map[string]any{"handler": "runsc"}, map[string]any{"handler": "kata"}) + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "gVisor") { + t.Errorf("runsc RuntimeClass not honoured:\n%s", got) + } + + // A microvm pool with no KVM node warns; a gvisor-less pool set does not + // demand gVisor. + f = healthyKube() + f.bodies[pathPools] = kubeList(kubePool("vm", 1, "microvm")) + f.bodies[pathSandboxes] = kubeList() + p, opts = clusterPreflighter(t, f) + got = strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "no node advertises ate.dev/kvm") || strings.Contains(got, "no gVisor") { + t.Errorf("microvm warnings:\n%s", got) + } + f.bodies[pathNodes] = kubeList(kubeNode("n1", "6.12.0", true)) + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "kvm") { + t.Errorf("KVM node not honoured:\n%s", got) + } +} + +func TestSubstratePreflightClusterFeatureGatesAndKernels(t *testing.T) { + f := healthyKube() + delete(f.bodies, substrateCertsGroupPath+"v1beta1") + f.bodies[pathNodes] = kubeList(kubeNode("old", "4.14.0-1", false), kubeNode("new", "5.15.0", false), kubeNode("odd", "weird", false)) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + for _, want := range []string{"PodCertificateRequest API is not served", "ClusterTrustBundle API is not served", "1 node(s) run a kernel older", "old (4.14.0-1)"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + + // GA v1 serving both resources is enough. + f = healthyKube() + delete(f.bodies, substrateCertsGroupPath+"v1beta1") + f.bodies[substrateCertsGroupPath+"v1"] = kubeResources("PodCertificateRequests", "ClusterTrustBundles") + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "not served") { + t.Errorf("v1 discovery not honoured:\n%s", got) + } +} + +// init pins the Kubernetes token path to a file that does not exist so the +// Control-API-only tests behave the same on a laptop and in a CI Pod (where +// the runner has a real ServiceAccount token). Tests that want the +// in-cluster halves inject kube/routerDial or call inClusterToken. +func init() { + substrateKubeTokenPath = filepath.Join(os.TempDir(), "hive-substrate-preflight-no-token-"+strconv.Itoa(os.Getpid())) +} + +// inClusterToken makes the prover believe it runs in a Pod wired for +// Substrate: a Kubernetes token and a projected Substrate token both exist. +// It returns the Substrate token path to set as opts.TokenFile. +func inClusterToken(t *testing.T) string { + t.Helper() + dir := t.TempDir() + prev := substrateKubeTokenPath + substrateKubeTokenPath = filepath.Join(dir, "k8s-token") + ate := filepath.Join(dir, "ate-token") + for _, f := range []string{substrateKubeTokenPath, ate} { + if err := os.WriteFile(f, []byte("t"), 0o600); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { substrateKubeTokenPath = prev }) + return ate +} + +func TestSubstratePreflightClusterSkippedOutsideCluster(t *testing.T) { + fake := &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil, "gvisor")}} + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(context.Background(), SubstrateOptions{Endpoint: "api:443"}); len(got) != 0 { + t.Errorf("outside a cluster the kube/router halves must be silent: %q", got) + } + + // In a Pod with no reachable API server the kube half reports once and + // the router half reports unusable credentials, instead of staying silent. + ate := inClusterToken(t) + // Only one of the two markers present is still "outside". + if substrateInCluster(SubstrateOptions{}) { + t.Error("a Kubernetes token alone must not count as in-cluster") + } + opts := SubstrateOptions{Endpoint: "api:443", TokenFile: ate, RouterClientBundle: filepath.Join(t.TempDir(), "missing.pem")} + got := p.probeCluster(context.Background(), opts) + if len(got) != 1 || !strings.Contains(got[0], "cluster facts could not be read") { + t.Errorf("in-cluster without an API server: %q", got) + } + if got := p.probeRouter(context.Background(), opts); len(got) != 1 || !strings.Contains(got[0], "not usable") { + t.Errorf("in-cluster without a bundle: %q", got) + } +} + +func TestSubstratePreflightKubeClientErrors(t *testing.T) { + k := &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: "/nonexistent"} + if _, err := k.get(context.Background(), "/x"); err == nil || !strings.Contains(err.Error(), "in-cluster CA") { + t.Errorf("missing CA: %v", err) + } + dir := t.TempDir() + bad := filepath.Join(dir, "ca.crt") + _ = os.WriteFile(bad, []byte("not pem"), 0o600) + k = &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: bad} + if _, err := k.get(context.Background(), "/x"); err == nil || !strings.Contains(err.Error(), "no certificates") { + t.Errorf("bad CA: %v", err) + } + _, ca, _ := selfSignedBundle(t, dir) + k = &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: ca} + if _, err := k.get(context.Background(), "/x"); err == nil { + t.Error("refused dial did not error") + } + if k.client == nil { + t.Error("client not cached") + } + k = &substrateKube{server: "http://[::1]:namedport", client: http.DefaultClient} + if _, err := k.get(context.Background(), "/x"); err == nil { + t.Error("bad URL did not error") + } +} + +func TestSubstrateOperatorNamespace(t *testing.T) { + for in, want := range map[string]string{ + "api.ate-system.svc:443": "ate-system", + "https://api.substrate-prod.svc.cluster.local": "substrate-prod", + "grpc://api.hive-sub.svc:8080": "hive-sub", + "": "ate-system", + "control.example.com:443": "ate-system", + "api..svc:443": "ate-system", + "not a url": "ate-system", + } { + if got := substrateOperatorNamespace(in); got != want { + t.Errorf("%q → %q, want %q", in, got, want) + } + } +} + +func TestSubstrateKernelLess(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"4.14.0-1", "4.14.77", true}, + {"4.14.77", "4.14.77", false}, + {"5.15.0-1.el8uek", "4.14.77", false}, + {"6.12.0-207.el9uek.aarch64", "4.14.77", false}, + {"4.9", "4.14.77", true}, + {"weird", "4.14.77", false}, + {"4.x.1", "4.14.77", false}, + {"5.15", "x", false}, + } + for _, c := range cases { + if got := substrateKernelLess(c.a, c.b); got != c.want { + t.Errorf("substrateKernelLess(%q, %q) = %v", c.a, c.b, got) + } + } +} + +// fakeRouterListener accepts TLS connections requiring a client certificate +// signed by the test CA and answers any CONNECT with 404, so the probe sees +// exactly what atenet-router would show an accepted or rejected client. +func fakeRouterListener(t *testing.T, cert tls.Certificate) net.Listener { + t.Helper() + pool := x509.NewCertPool() + leaf, _ := x509.ParseCertificate(cert.Certificate[0]) + pool.AddCert(leaf) + cfg := &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool} + ln, err := tls.Listen("tcp", "127.0.0.1:0", cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + if _, err := http.ReadRequest(bufio.NewReader(c)); err != nil { + return + } + _, _ = io.WriteString(c, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + }() + } + }() + return ln +} + +func TestSubstratePreflightRouter(t *testing.T) { + dir := t.TempDir() + bundle, ca, cert := selfSignedBundle(t, dir) + token := filepath.Join(dir, "token") + _ = os.WriteFile(token, []byte("t"), 0o600) + ln := fakeRouterListener(t, cert) + kube := &substrateKube{server: "http://127.0.0.1:1", tokenPath: token, caPath: ca, client: http.DefaultClient} + p := &SubstratePreflighter{kube: kube} + ctx := context.Background() + + // Good bundle, router verifying it → quiet. + opts := SubstrateOptions{RouterURL: "https://" + ln.Addr().String(), RouterClientBundle: bundle, RouterCAFile: ca} + if got := p.probeRouter(ctx, opts); len(got) != 0 { + t.Errorf("healthy router warned: %q", got) + } + + // Unusable bundle → credentials warning. + bad := opts + bad.RouterClientBundle = filepath.Join(dir, "missing.pem") + if got := p.probeRouter(ctx, bad); len(got) != 1 || !strings.Contains(got[0], "client credentials are not usable") { + t.Errorf("missing bundle: %q", got) + } + + // Router rejects the client certificate: a second CA the router does + // not trust. TLS 1.3 surfaces that on the first read, not the handshake. + otherDir := t.TempDir() + otherBundle, _, _ := selfSignedBundle(t, otherDir) + rejected := opts + rejected.RouterClientBundle = otherBundle + if got := p.probeRouter(ctx, rejected); len(got) != 1 || !strings.Contains(got[0], "rejected hive's client certificate") || !strings.Contains(got[0], otherBundle) { + t.Errorf("rejected cert: %q", got) + } + + // Unreachable router. + down := opts + down.RouterURL = "https://127.0.0.1:1" + if got := p.probeRouter(ctx, down); len(got) != 1 || !strings.Contains(got[0], "is unreachable") { + t.Errorf("unreachable: %q", got) + } + + // Injected dial wins over the real one; default bundle path is named + // when none is configured. + stub := &SubstratePreflighter{kube: kube, routerDial: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("stubbed") }} + if got := stub.probeRouter(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "stubbed") { + t.Errorf("stubbed dial: %q", got) + } + if substrateRouterBundlePath(SubstrateOptions{}) != SubstrateDefaultRouterClientBundle { + t.Error("default bundle path") + } + + // Outside a cluster the router half is silent. + if got := (&SubstratePreflighter{}).probeRouter(ctx, opts); len(got) != 0 { + t.Errorf("outside cluster: %q", got) + } +} diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go index 389ffabb18..653616887e 100644 --- a/src/crux/sandbox/substratebridge/bridge.go +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -100,9 +100,28 @@ func (h *Handler) workspace() string { return DefaultWorkspace } +// clearDir empties dir without removing dir itself: inside the actor the +// workspace is a durableDir mount point, and unlinking a mount point fails +// with EBUSY. +func clearDir(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + func (h *Handler) putWorkspace(w http.ResponseWriter, r *http.Request) { dir := h.workspace() - if err := os.RemoveAll(dir); err != nil { + if err := clearDir(dir); err != nil { http.Error(w, fmt.Sprintf("substrate bridge: clearing %s: %v", dir, err), http.StatusInternalServerError) return } diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go index e91754700a..bc18a77a56 100644 --- a/src/crux/sandbox/substratebridge/bridge_test.go +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -392,3 +392,50 @@ func TestServeStopsOnContextAndListenerClose(t *testing.T) { t.Errorf("Serve returned nil after the listener closed") } } + +// Inside the actor /workspace is a durableDir mount point; PUT must empty it +// in place rather than unlink it (which fails with EBUSY on a mount). +func TestPutWorkspaceClearsInPlace(t *testing.T) { + ws := t.TempDir() + writeFile(t, filepath.Join(ws, "stale.txt"), "old") + writeFile(t, filepath.Join(ws, "nested", "deep.txt"), "old") + before, err := os.Stat(ws) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(&Handler{Workspace: ws}) + defer srv.Close() + body := tarStream(t, func(tw *tar.Writer) { + content := []byte("new") + _ = tw.WriteHeader(&tar.Header{Name: "fresh.txt", Mode: 0o644, Size: int64(len(content)), Typeflag: tar.TypeReg}) + _, _ = tw.Write(content) + }) + req, _ := http.NewRequest(http.MethodPut, srv.URL+PathWorkspace, body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + t.Fatalf("put = %d", resp.StatusCode) + } + after, err := os.Stat(ws) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Error("workspace directory was recreated; it must be emptied in place") + } + if _, err := os.Stat(filepath.Join(ws, "stale.txt")); !os.IsNotExist(err) { + t.Error("stale.txt survived PUT") + } + if _, err := os.Stat(filepath.Join(ws, "nested")); !os.IsNotExist(err) { + t.Error("nested/ survived PUT") + } + if got, err := os.ReadFile(filepath.Join(ws, "fresh.txt")); err != nil || string(got) != "new" { + t.Errorf("fresh.txt = %q, %v", got, err) + } + if err := clearDir(filepath.Join(ws, "does-not-exist")); err != nil { + t.Errorf("clearDir of a missing dir = %v, want nil", err) + } +} diff --git a/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml b/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml new file mode 100644 index 0000000000..7a60c377ee --- /dev/null +++ b/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml @@ -0,0 +1,72 @@ +# Lets the spoke's Substrate preflight (sandbox.runtime: substrate, #17698; +# crux/sandbox/substrate_preflight_cluster.go) read the cluster facts behind +# a failing kick: the operator's CRDs and workloads, WorkerPools and +# SandboxConfigs, RuntimeClasses, node kernels/KVM, and whether the +# certificates.k8s.io APIs behind the PodCertificateRequest and +# ClusterTrustBundle feature gates are served. Read-only, get/list only; +# without it the preflight emits one warning and skips these checks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hive-substrate-preflight + labels: + app.kubernetes.io/name: hive +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["workerpools.ate.dev", "sandboxconfigs.ate.dev"] + verbs: ["get"] + - apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs"] + verbs: ["get", "list"] + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hive-substrate-preflight + labels: + app.kubernetes.io/name: hive +subjects: + - kind: ServiceAccount + name: hive + namespace: hive +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hive-substrate-preflight +--- +# The operator namespace (ate-system by default; the preflight derives it +# from sandbox.substrate.endpoint's api..svc host). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hive-substrate-preflight + namespace: ate-system + labels: + app.kubernetes.io/name: hive +rules: + - apiGroups: ["apps"] + resources: ["deployments", "daemonsets"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hive-substrate-preflight + namespace: ate-system + labels: + app.kubernetes.io/name: hive +subjects: + - kind: ServiceAccount + name: hive + namespace: hive +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hive-substrate-preflight From e8f0a2e00d6ecfae961ac3b7830ced3cf7c5bba4 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 15:54:51 -0700 Subject: [PATCH 017/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openshell=20sandb?= =?UTF-8?q?ox=20names=20and=20driver=5Fconfig=20fit=20gateway=200.1.8=20li?= =?UTF-8?q?mits=20(#17780)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prototype on LKE (#17643) hit four gateway rejections before a kick ran: sandbox names over MAX_ROUTABLE_NAME_LEN (19), the reserved 'workspace' volume name, default-denied pod labels/annotations, and the missing Agent Sandbox prerequisite. Names are now hk-- (fnv tag when the slug does not fit), the claim mounts as hive-workspace, and the deploy doc records the operator settings the measurements depended on (allowed_pod_labels, workspaceStorageClass, sandboxUid, auth modes) plus the landlock-in-LSM preflight lesson and the measured numbers. Refs #17643, #17642 Signed-off-by: Andrew Anderson --- changelog.d/fixed-openshell-sandbox-name.md | 1 + src/crux/sandbox/openshell.go | 42 +++++++++++--- src/crux/sandbox/openshell_logs.go | 2 +- src/crux/sandbox/openshell_logs_test.go | 2 +- src/crux/sandbox/openshell_test.go | 29 ++++++++-- src/deploy/openshell.md | 55 +++++++++++++++---- .../adr/0026-sandbox-runtime-selection.md | 2 +- src/docs/sandbox-isolation.md | 4 +- 8 files changed, 107 insertions(+), 30 deletions(-) create mode 100644 changelog.d/fixed-openshell-sandbox-name.md diff --git a/changelog.d/fixed-openshell-sandbox-name.md b/changelog.d/fixed-openshell-sandbox-name.md new file mode 100644 index 0000000000..6f902b3cbd --- /dev/null +++ b/changelog.d/fixed-openshell-sandbox-name.md @@ -0,0 +1 @@ +- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) diff --git a/src/crux/sandbox/openshell.go b/src/crux/sandbox/openshell.go index 5da7dfe449..f8d7cd0cdc 100644 --- a/src/crux/sandbox/openshell.go +++ b/src/crux/sandbox/openshell.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "hash/fnv" "net/url" "os" "path/filepath" @@ -50,14 +51,25 @@ const ( // on the created sandbox. OpenShellPolicyProfileAnnotation = "hive.dev/policy-profile" - openshellNamePrefix = "hive-kick-" - openshellNameMax = 63 + // openshellNamePrefix and openshellNameMax follow the gateway's + // MAX_ROUTABLE_NAME_LEN (19): a sandbox name is one of three segments + // in a 63-char DNS label (----), so the + // prefix is short and the timestamp is base36 (measured on 0.1.8, + // hivecommons/hive#17643). + openshellNamePrefix = "hk-" + openshellNameMax = 19 + openshellNameUnixBase = 36 + openshellNameTagLen = 3 + openshellNameTagMod = 36 * 36 * 36 openshellRequestWait = 30 * time.Second openshellReadyPoll = 2 * time.Second openshellMaxOutput = 16 << 20 openshellManagedLabel = "app.kubernetes.io/managed-by" openshellManagedValue = "hive-openshell" - openshellHTTPSPort = 443 + // openshellWorkspaceVolume names hive's claim in driver_config; the + // gateway reserves the bare name "workspace" for its own volumes. + openshellWorkspaceVolume = "hive-workspace" + openshellHTTPSPort = 443 ) // OpenShellAlwaysAllowedHosts are the egress hosts every hive sandbox gets: @@ -361,12 +373,12 @@ func (l *OpenShellLauncher) execEnv(spec LaunchSpec) map[string]string { func (l *OpenShellLauncher) driverConfig(subPath, mount string, readOnly bool) map[string]any { k8s := map[string]any{ "volumes": []any{map[string]any{ - "name": "workspace", + "name": openshellWorkspaceVolume, "persistent_volume_claim": map[string]any{"claim_name": l.Options.WorkspaceClaim, "read_only": false}, }}, "containers": map[string]any{"agent": map[string]any{ "volume_mounts": []any{map[string]any{ - "name": "workspace", + "name": openshellWorkspaceVolume, "mount_path": mount, "sub_path": subPath, "read_only": readOnly, @@ -550,16 +562,28 @@ func openshellWorkspaceSubPath(workspace, claimMount string) (string, error) { var openshellNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) // openshellSandboxName builds a unique, DNS-label-safe sandbox name from the -// launch name and a timestamp: hive-kick--. +// launch name and a timestamp: hk--, at most +// openshellNameMax characters. The launch name's hive-/-sandbox wrapping is +// dropped so the agent slug gets the room; a slug that still does not fit is +// cut and tagged with a short hash of the whole slug, so two agents sharing +// a long prefix never map to the same sandbox name. func openshellSandboxName(launchName string, now time.Time) string { - base := strings.Trim(openshellNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + base := strings.ToLower(strings.TrimSuffix(launchName, launchNameSuffix)) + base = strings.TrimPrefix(base, launchNamePrefix) + base = strings.Trim(openshellNameUnsafe.ReplaceAllString(base, "-"), "-") if base == "" { base = "agent" } - suffix := fmt.Sprintf("-%d", now.Unix()) + suffix := "-" + strconv.FormatInt(now.Unix(), openshellNameUnixBase) room := openshellNameMax - len(openshellNamePrefix) - len(suffix) if len(base) > room { - base = strings.TrimRight(base[:room], "-") + h := fnv.New32a() + _, _ = h.Write([]byte(base)) + tag := strconv.FormatUint(uint64(h.Sum32())%openshellNameTagMod, openshellNameUnixBase) + for len(tag) < openshellNameTagLen { + tag = "0" + tag + } + base = strings.TrimRight(base[:room-openshellNameTagLen], "-") + tag } return openshellNamePrefix + base + suffix } diff --git a/src/crux/sandbox/openshell_logs.go b/src/crux/sandbox/openshell_logs.go index 4913ebb4ed..191c1a0a65 100644 --- a/src/crux/sandbox/openshell_logs.go +++ b/src/crux/sandbox/openshell_logs.go @@ -153,7 +153,7 @@ func openshellSandboxNameUnix(launchName, name string) (int64, bool) { if i < 0 || !strings.HasPrefix(name, openshellNamePrefix) { return 0, false } - unix, err := strconv.ParseInt(name[i+1:], 10, 64) + unix, err := strconv.ParseInt(name[i+1:], openshellNameUnixBase, 64) if err != nil { return 0, false } diff --git a/src/crux/sandbox/openshell_logs_test.go b/src/crux/sandbox/openshell_logs_test.go index 2da1939117..b688e9b9e1 100644 --- a/src/crux/sandbox/openshell_logs_test.go +++ b/src/crux/sandbox/openshell_logs_test.go @@ -270,7 +270,7 @@ func TestOpenShellSandboxNameUnix(t *testing.T) { {good, true}, {"nodash", false}, {"other-deps-1700000000", false}, - {openshellNamePrefix + "deps-notanumber", false}, + {openshellNamePrefix + "deps-not.a.number", false}, {openshellSandboxName(LaunchName("deps-bot"), time.Unix(1700000000, 0)), false}, } for _, c := range cases { diff --git a/src/crux/sandbox/openshell_test.go b/src/crux/sandbox/openshell_test.go index 6f20cfc2d6..864eb64cf9 100644 --- a/src/crux/sandbox/openshell_test.go +++ b/src/crux/sandbox/openshell_test.go @@ -145,7 +145,7 @@ func TestOpenShellRunSuccess(t *testing.T) { if c.workspace != OpenShellDefaultWorkspace { t.Errorf("workspace = %q", c.workspace) } - if c.name != "hive-kick-deps-1700000000" { + if c.name != "hk-deps-s44we8" { t.Errorf("name = %q", c.name) } if c.spec.Template.Image != "registry.example/dev:sha256-abc" { @@ -186,12 +186,17 @@ func TestOpenShellRunDriverConfigMountsWorkspaceClaim(t *testing.T) { t.Fatalf("driver config = %v", dc) } vols := k8s["volumes"].([]any) - pvc := vols[0].(map[string]any)["persistent_volume_claim"].(map[string]any) + vol := vols[0].(map[string]any) + // The gateway rejects a caller volume named "workspace" (reserved). + if vol["name"] != openshellWorkspaceVolume || vol["name"] == "workspace" { + t.Errorf("volume name = %v", vol["name"]) + } + pvc := vol["persistent_volume_claim"].(map[string]any) if pvc["claim_name"] != "hive-data" { t.Errorf("claim = %v", pvc) } mount := k8s["containers"].(map[string]any)["agent"].(map[string]any)["volume_mounts"].([]any)[0].(map[string]any) - if mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { + if mount["name"] != vol["name"] || mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { t.Errorf("mount = %v", mount) } pod := k8s["pod"].(map[string]any) @@ -362,16 +367,28 @@ func TestOpenShellAddress(t *testing.T) { func TestOpenShellSandboxName(t *testing.T) { now := time.Unix(1700000000, 0) - if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hive-kick-deps-agent-1700000000" { + if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hk-deps-aftv-s44we8" { t.Errorf("name = %q", got) } long := openshellSandboxName(strings.Repeat("a", 100), now) - if len(long) > openshellNameMax || !strings.HasSuffix(long, "-1700000000") { + // The gateway rejects routable names over 19 characters + // (MAX_ROUTABLE_NAME_LEN); the suffix is unix seconds in base36. + if len(long) != openshellNameMax || !strings.HasSuffix(long, "-s44we8") { t.Errorf("long name = %q (%d)", long, len(long)) } - if got := openshellSandboxName("---", now); got != "hive-kick-agent-1700000000" { + if got := openshellSandboxName("---", now); got != "hk-agent-s44we8" { t.Errorf("empty base = %q", got) } + // The launch name's hive-/-sandbox wrapping is dropped, and two agents + // that share a long prefix get distinct, stable names. + if got := openshellSandboxName(LaunchName("deps"), now); got != "hk-deps-s44we8" { + t.Errorf("launch name = %q", got) + } + one := openshellSandboxName(LaunchName("reviewer-one"), now) + two := openshellSandboxName(LaunchName("reviewer-two"), now) + if one == two || len(one) != openshellNameMax || one != openshellSandboxName(LaunchName("reviewer-one"), now) { + t.Errorf("truncated names = %q %q", one, two) + } } func TestOpenShellRunRefusesClineProvider(t *testing.T) { diff --git a/src/deploy/openshell.md b/src/deploy/openshell.md index 7653c1ad9f..706461a648 100644 --- a/src/deploy/openshell.md +++ b/src/deploy/openshell.md @@ -16,7 +16,8 @@ record it alongside the hive config. | Requirement | Why | How to check | |---|---|---| -| Node kernel **≥ 6.2** with Landlock ABI ≥ 3 | OpenShell's sandbox qualification fails below it | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`. Of the clusters hive runs on, LKE (Debian 13, 6.12) qualifies; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| Node kernel **≥ 6.2** with Landlock **active** | OpenShell's sandbox qualification fails below it; a new-enough kernel built without Landlock in the LSM list fails the same way | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`, then from a privileged debug pod `cat /sys/kernel/security/lsm` must contain `landlock`. Of the clusters hive runs on, LKE (Debian 13, 6.12) and OKE on OL9/UEK8 (6.12) qualify; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| The **Agent Sandbox** controller (`sandboxes.agents.x-k8s.io`) | the gateway's Kubernetes driver creates `Sandbox` objects; the chart's preflight refuses to install without the API | `kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/sandbox.yaml`, then `kubectl api-resources \| grep agents.x-k8s.io` | | A CNI that **enforces** egress `NetworkPolicy` (Calico, Cilium) | the gateway proxies egress; the pod must not be able to go around it. Flannel accepts policies and ignores them | create a deny-all policy in a scratch namespace and confirm a connection is actually refused, not just that the object exists | | Cluster-admin for the install | CRDs, the gateway's RBAC, PKI secrets | — | | A **ReadWriteMany** claim for hive's data when sandboxes may land on other nodes | the workspace is shared by mounting the same PVC | same caveat as the Job runtime | @@ -29,9 +30,17 @@ helm upgrade --install openshell \ --version \ --namespace openshell --create-namespace \ --set server.drivers.kubernetes.allowDriverConfig=true \ - --set-string 'server.env.OPENSHELL_TELEMETRY_ENABLED=false' + --set server.telemetryEnabled=false \ + --set server.workspaceStorageClass= \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.allowed_pod_labels={app.kubernetes.io/managed-by}' \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.allowed_pod_annotations={hive.dev/*}' ``` +`gatewayConfig` is rendered to TOML: its top-level keys are table names +written as dotted strings (`openshell.drivers.kubernetes:` in a values file, +escaped as above on the command line), not nested maps; nesting +`openshell: {drivers: …}` breaks the config and crash-loops the gateway. + Then, for the workspace hive will use (hive's `agent_sandbox.openshell.namespace`, default `default`), install the workspace chart with driver config allowed so the gateway may read PVC metadata for admission: @@ -61,12 +70,30 @@ Settings hive's runtime depends on, in gateway terms: `kubectl label pvc hive-data openshell.ai/sandbox-attachable=true openshell.ai/sandbox-attachable-workspace=default` on the spoke's namespace is enough for the defaults. -- **`OPENSHELL_TELEMETRY_ENABLED=false`** on the gateway. Hive's threat model +- **`allowed_pod_labels` / `allowed_pod_annotations`** under + `[openshell.drivers.kubernetes]`. The gateway rejects any template label or + annotation it was not told about; hive labels sandboxes + `app.kubernetes.io/managed-by=hive-openshell` and annotates them + `hive.dev/policy-profile` when a profile is set. +- **`server.workspaceStorageClass`**. The gateway provisions a 2Gi + `workspace-` PVC for every sandbox even though hive mounts its own + claim. On a block-storage default class that provision + attach added ~20 s + to every kick on LKE (27 s vs 9 s warm start, [#17643](https://github.com/hivecommons/hive/issues/17643)); + point it at an NFS/RWX or otherwise fast class. +- **Sandbox uid**. The gateway runs every sandbox as `runAsUser`/`fsGroup` + `10001` by default, overriding the image's `USER`. The kick workspace hive + creates on the shared claim must be writable by that uid: either set + `server.sandboxUid`/`server.sandboxGid` to the uid crux runs as, or make the + claim's workspace directories group-writable for 10001. +- **`server.telemetryEnabled=false`** on the gateway. Hive's threat model treats sandbox metadata as spoke-private; keep upstream telemetry off. -- **A bearer token** for hive. Create one per the upstream access-control page - and hand it to the hive pod as an environment variable (a Secret-backed - `env` entry on the hive Deployment); name that variable in - `agent_sandbox.openshell.token_env`. The token never goes in `hive.yaml`. +- **A bearer token** for hive. Upstream authenticates clients by mTLS, OIDC + JWT, or a trusted reverse proxy; hive's client sends a static bearer token, + so "token" here means a JWT from the OIDC issuer the gateway trusts (see the + upstream access-control page). Hand it to the hive pod as an environment + variable (a Secret-backed `env` entry on the hive Deployment); name that + variable in `agent_sandbox.openshell.token_env`. The token never goes in + `hive.yaml`. `allowUnauthenticatedUsers` is for throwaway clusters only. - **Digest-pinned images**: the agent's `sandbox.image` should be `registry/image@sha256:…`. The gateway's own supervisor and sandbox runtime images are pinned by the chart version. @@ -91,11 +118,19 @@ Settings hive's runtime depends on, in gateway terms: `api.github.com` must be **denied** and `git fetch` must succeed. Sandboxes are labelled `app.kubernetes.io/managed-by=hive-openshell` and are deleted when the kick ends; one that is still there after a finished kick is a bug. + Sandbox names are `hk--` (the gateway caps names at + 19 characters) and the claim is mounted as volume `hive-workspace` + (`workspace` is reserved by the gateway). +5. Agent images must ship a glibc `sh`: debian/ubuntu/fedora/EL-based images + and hive's own image work, but busybox/musl images (alpine) fail at + `spawn sandbox entrypoint process '/bin/sh': Permission denied`. ## Not covered Hive does not preflight any of the above ([#17644](https://github.com/hivecommons/hive/issues/17644)) and has no Podman-driver path for standalone hosts yet. The hub provisioner -does not install the gateway for hosted spokes; that is a follow-up once the -prototype numbers ([#17643](https://github.com/hivecommons/hive/issues/17643)) -are in. +does not install the gateway for hosted spokes; that is a follow-up. The +prototype numbers are on [#17643](https://github.com/hivecommons/hive/issues/17643): +on LKE with the settings above, cold start 10.0 s vs 6.0 s for the Job runtime +(1.67×), warm 9.1 s vs 3.8 s (2.4×), +38 Mi per sandbox, egress policy +enforced. diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index 153111ef6f..b2d0a8c838 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -1,6 +1,6 @@ # ADR-0026: Sandbox runtime selection -Status: Accepted (phase 1 OpenShell implemented, runtime opt-in — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) +Status: Accepted (phase 1 OpenShell implemented, runtime opt-in; measured on LKE in #17643: cold 1.67× / warm 2.4× the Job runtime, egress enforced — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) ## Context diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 507b404980..13199d6fee 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -276,11 +276,11 @@ The gateway, workspace, token, TLS and claim fields are cluster facts and fall b **The credential boundary.** Same rule as the Job runtime: environment variables from the allowlist are copied in with the same credential-name filter, hive's brokered GitHub token never enters the sandbox, and hive pushes and opens the PR from the returned workspace. There is **no** Secret-reference path into the sandbox in phase 1, so model credentials have to arrive through an allowlisted non-credential variable pointing at a routable gateway named in `allow_hosts`. The Cline backend, which needs a provider file or Secret, is refused on this runtime. -**Lifecycle.** The sandbox is named `hive-kick--` and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. +**Lifecycle.** The sandbox is named `hk--` (at most 19 characters, the gateway's limit for DNS-routable names) and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. **What hive checks, and what it cannot.** Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when an openshell-runtime agent resolves no `gateway_url` or no `workspace_claim`, and refuses the kick with the same message before the agent is marked running. It also preflights each distinct gateway (`Health().Check` and `Config().GetGateway`, read-only, cached for five minutes, bounded to 30 s) and adds a warning when the gateway is unreachable or unhealthy, `allow_driver_config` is not enabled, or telemetry is on. It cannot see the node kernel, the CNI or the claim's labels from inside its pod; those remain the operator prerequisites in [deploy/openshell.md](../deploy/openshell.md). -**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hive-kick--` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. +**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hk--` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. **How to verify.** With a gateway reachable and the claim attachable, run the env-gated integration test from a pod that has the claim mounted: `OPENSHELL_GATEWAY_URL=… OPENSHELL_WORKSPACE_CLAIM=hive-data OPENSHELL_IMAGE= go test ./crux/sandbox -run TestOpenShellIntegration -v`. It writes a file through `/workspace`, reads it back on the hive side, and logs the round-trip latency. Then run a kick and confirm in the sandbox log that a `POST` to `api.github.com` is denied while `git fetch` succeeds — that denial is the whole point of the runtime. From aaa74123d740030cefc9493dff982e5bf174aefe Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 23:09:29 +0000 Subject: [PATCH 018/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.222.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.222.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 9 +++++++++ changelog.d/fixed-effect-ledger-unknown-op-reconcile.md | 1 - changelog.d/fixed-gfi-archived-repositories.md | 1 - changelog.d/fixed-stage-engine-nil-config.md | 1 - .../fixed-unauthorized-label-removal-readonly-lane.md | 1 - 5 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/fixed-effect-ledger-unknown-op-reconcile.md delete mode 100644 changelog.d/fixed-gfi-archived-repositories.md delete mode 100644 changelog.d/fixed-stage-engine-nil-config.md delete mode 100644 changelog.d/fixed-unauthorized-label-removal-readonly-lane.md diff --git a/CHANGELOG.md b/CHANGELOG.md index cb85f26dab..8b3aa3ce8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.222.1) + +### Fixed + +- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). +- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. +- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). +- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). + ## 2026-11-11 (v5.222.0) ### Added diff --git a/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md b/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md deleted file mode 100644 index cd12604414..0000000000 --- a/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md +++ /dev/null @@ -1 +0,0 @@ -- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). diff --git a/changelog.d/fixed-gfi-archived-repositories.md b/changelog.d/fixed-gfi-archived-repositories.md deleted file mode 100644 index 4d7c472e30..0000000000 --- a/changelog.d/fixed-gfi-archived-repositories.md +++ /dev/null @@ -1 +0,0 @@ -- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. diff --git a/changelog.d/fixed-stage-engine-nil-config.md b/changelog.d/fixed-stage-engine-nil-config.md deleted file mode 100644 index 25490b8df1..0000000000 --- a/changelog.d/fixed-stage-engine-nil-config.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). diff --git a/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md b/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md deleted file mode 100644 index 9939b3a538..0000000000 --- a/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md +++ /dev/null @@ -1 +0,0 @@ -- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). From e0e9c0c5cf40660024c0826752b139a982268397 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:21:06 -0700 Subject: [PATCH 019/223] =?UTF-8?q?=F0=9F=90=9B=20marketplace:=20stream=20?= =?UTF-8?q?repository=20tarballs=20and=20keep=20only=20pipeline=20files=20?= =?UTF-8?q?(#17793)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The marketplace fetcher extracted each default-branch tarball to disk via complexity.ExtractTarGz under a 256 MiB whole-tree cap, so large monorepos (kubernetes/kubernetes, kserve/kserve, meshery/meshery, shipwright-io/build, spidernet-io/spiderpool) failed with "archive exceeds size cap" and never reached the catalogue. ingest.ReadTarball now streams the archive and keeps only the paths ReadTree would select (shared selectTreeFile), 1 MiB per file and 32 MiB in total, discarding everything else without buffering; the complexity cap is untouched for its own callers. GET /api/marketplace/status now includes failing:[{repo,error,checkedAt}] without ?verbose=1 (repo names and fetch errors carry no secrets), so an operator with a plain PAT can see why repos fail. Four corpus entries that GitHub now redirects are updated to their current names. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../fixed-marketplace-stream-tarball.md | 1 + src/docs/api-reference/hub-server.md | 2 +- src/docs/design/workflow-models.md | 6 +- src/pkg/hub/marketplace.go | 13 +- src/pkg/hub/marketplace_refresh.go | 13 +- src/pkg/hub/marketplace_test.go | 7 +- .../workflowmodel/ingest/corpus/corpus.json | 8 +- src/pkg/workflowmodel/ingest/tree.go | 130 +++++++++++++++--- src/pkg/workflowmodel/ingest/tree_test.go | 69 ++++++++++ 9 files changed, 203 insertions(+), 46 deletions(-) create mode 100644 changelog.d/fixed-marketplace-stream-tarball.md diff --git a/changelog.d/fixed-marketplace-stream-tarball.md b/changelog.d/fixed-marketplace-stream-tarball.md new file mode 100644 index 0000000000..fd6a16ff29 --- /dev/null +++ b/changelog.d/fixed-marketplace-stream-tarball.md @@ -0,0 +1 @@ +- Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index b79feb07b9..0c4f7404d3 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -75,4 +75,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | | `POST` | `/api/marketplace/refresh` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}`. Spokes are rate-limited across all hives (one full run per hour, 10 `?repo=` runs per minute) → `429` + `Retry-After`; admins are not. `401` anonymous, `403` non-admin session or unknown hive | `pkg/hub/marketplace.go:39` | -| `GET` | `/api/marketplace/status` | Public; `?verbose=1` requires spoke heartbeat bearer + `X-Hive-ID` or Hub admin session | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, pendingRepos, lastRefreshErrors, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary (persisted across restarts) and counts by tier. `?verbose=1` adds `failing:[{repo, error, checkedAt}]` and `pending:[repo]` (corpus repos never ingested) | `pkg/hub/marketplace.go:40` | +| `GET` | `/api/marketplace/status` | Public; `?verbose=1` requires spoke heartbeat bearer + `X-Hive-ID` or Hub admin session | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, failing:[{repo, error, checkedAt}], pendingRepos, lastRefreshErrors, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary (persisted across restarts), counts by tier and the failing repos with their fetch errors (no secrets). `?verbose=1` adds `pending:[repo]` (corpus repos never ingested) | `pkg/hub/marketplace.go:40` | diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index 10ed66c3ef..f025f9276b 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -736,7 +736,7 @@ Routes (see [hub-server.md](../api-reference/hub-server.md)): | `GET /api/marketplace/workflows/{name}/{version}/bundle` | public | one bundle, `Cache-Control: immutable` | | `POST /api/marketplace/workflows` | spoke bearer or hub admin | publish; hub re-verifies | | `POST /api/marketplace/refresh[?repo=]` | spoke bearer or hub admin | on-demand ingest; spokes rate-limited (`429`) | -| `GET /api/marketplace/status[?verbose=1]` | public (`verbose`: spoke bearer or hub admin) | last refresh, counts by tier; `verbose` adds failing and pending repos | +| `GET /api/marketplace/status[?verbose=1]` | public (`verbose`: spoke bearer or hub admin) | last refresh, counts by tier, failing repos with their errors; `verbose` adds pending repos | **Who may publish what.** A spoke authenticates with its heartbeat bearer bound to the `X-Hive-ID` header (the same per-hive key the heartbeat and NPS lanes verify) and may @@ -748,7 +748,9 @@ same-origin CSRF evidence. The hub never stores a bundle it did not itself `Veri `pkg/workflowmodel/ingest/corpus` (262 projects / 299 repositories; `cmd/workflow-survey` falls back to it when no `projects.json` is given) plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch -tarball through its GitHub identity, keeps only pipeline files (`ingest.ReadTree`), and +tarball through its GitHub identity and streams it (`ingest.ReadTarball`): only the paths +`ingest.ReadTree` would select are kept (per file ≤ 1 MiB, ≤ 32 MiB in total), everything +else is discarded as it streams, so a monorepo's source size never matters. It then records `{format, sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → `ingest.Lifecycle` → `bundle.Export` → publish as `ingested-YYYYMMDD.N`, tier `ingested-unreviewed`, with the source SHA in an annotation. Curated versions are never diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 73e5196ae1..19750e96aa 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -19,8 +19,8 @@ import ( // GET /api/marketplace/workflows/{name}/{version}/bundle one bundle // POST /api/marketplace/workflows publish (spoke bearer or admin) // POST /api/marketplace/refresh[?repo=] on-demand ingest (spoke bearer or admin) -// GET /api/marketplace/status[?verbose=1] last refresh + counts by tier -// (verbose: failing/pending repos; spoke bearer or admin) +// GET /api/marketplace/status[?verbose=1] last refresh + counts by tier + failing repos +// (verbose: pending repos; spoke bearer or admin) // // Tier policy: a publisher may assert `local`, `local-published` or // `ingested-unreviewed`; `curated` and `built-in` require a hub admin @@ -296,9 +296,10 @@ type marketplaceFailingRepo struct { CheckedAt time.Time `json:"checkedAt"` } -// handleMarketplaceStatus reports the last refresh and tier counts. -// ?verbose=1 (spoke bearer or admin) adds the failing repositories with -// their error strings and the repositories still awaiting a first ingest. +// handleMarketplaceStatus reports the last refresh and tier counts, plus +// the failing repositories with their error strings (repo names and fetch +// errors carry no secrets, so any caller sees them). ?verbose=1 (spoke +// bearer or admin) adds the repositories still awaiting a first ingest. func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return @@ -358,11 +359,11 @@ func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Reque if !lastChecked.IsZero() { out["lastCheckedAt"] = lastChecked } + out["failing"] = failing if verbose { if pending == nil { pending = []string{} } - out["failing"] = failing out["pending"] = pending } w.Header().Set("Content-Type", "application/json") diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index ed164784ce..c6eb57af02 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/evidence/complexity" "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/bundle" "github.com/hivecommons/hive/pkg/workflowmodel/ingest" @@ -47,7 +46,6 @@ const ( // continuing. marketplaceRateFloor = 50 marketplaceRateMaxWait = 10 * time.Minute - marketplaceMaxTarball = 256 << 20 marketplaceSeedDelay = 30 * time.Second // marketplaceStaleAfter is how old a repo's refresh record may be // before the startup pass re-ingests it: the nightly interval, so a @@ -377,16 +375,7 @@ func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) ( _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) return nil, "", fmt.Errorf("tarball: %s", resp.Status) } - dir, err := os.MkdirTemp("", "hive-marketplace-") - if err != nil { - return nil, "", err - } - defer os.RemoveAll(dir) - top, err := complexity.ExtractTarGz(resp.Body, dir, marketplaceMaxTarball) - if err != nil { - return nil, "", err - } - files, err := ingest.ReadTree(top) + files, err := ingest.ReadTarball(resp.Body) if err != nil { return nil, "", err } diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index 185a7596a6..c3672cd46e 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -594,8 +594,11 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if status["pendingRepos"].(float64) != 0 || status["lastRefreshErrors"].(float64) != 0 { t.Fatalf("pending/lastErrors: %s", rec.Body.String()) } - if _, ok := status["failing"]; ok { - t.Fatal("non-verbose status must not list failing repos") + if f, ok := status["failing"].([]interface{}); !ok || len(f) != 0 { + t.Fatalf("non-verbose status must list failing repos (empty here): %s", rec.Body.String()) + } + if _, ok := status["pending"]; ok { + t.Fatal("non-verbose status must not list pending repos") } byTier := status["byTier"].(map[string]interface{}) if byTier[bundle.TierIngestedUnreviewed].(float64) != 1 { diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus.json b/src/pkg/workflowmodel/ingest/corpus/corpus.json index cfe16ee3bf..c831ae57ee 100644 --- a/src/pkg/workflowmodel/ingest/corpus/corpus.json +++ b/src/pkg/workflowmodel/ingest/corpus/corpus.json @@ -306,7 +306,7 @@ "maturity": "sandbox", "category": "Runtime", "repos": [ - "containers/composefs" + "composefs/composefs" ] }, { @@ -1120,7 +1120,7 @@ "maturity": "sandbox", "category": "AI Native Infra", "repos": [ - "ray-project/kuberay" + "kuberay-project/kuberay" ] }, { @@ -1542,7 +1542,7 @@ "maturity": "archived", "category": "Observability and Analysis", "repos": [ - "OpenObservability/OpenMetrics" + "prometheus/OpenMetrics" ] }, { @@ -1738,7 +1738,7 @@ "maturity": "sandbox", "category": "Provisioning", "repos": [ - "ratify-project/ratify" + "notaryproject/ratify" ] }, { diff --git a/src/pkg/workflowmodel/ingest/tree.go b/src/pkg/workflowmodel/ingest/tree.go index fc9beafd7a..87546ba559 100644 --- a/src/pkg/workflowmodel/ingest/tree.go +++ b/src/pkg/workflowmodel/ingest/tree.go @@ -1,6 +1,11 @@ package ingest import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" "io/fs" "os" "path/filepath" @@ -50,28 +55,13 @@ func ReadTree(root string) (map[string][]byte, error) { if !d.Type().IsRegular() || len(files) >= MaxTreeFiles { return nil } - switch { - case IsIntakePath(rel): - case !isYAMLPath(rel): - return nil - case githubactions.IsWorkflowPath(rel), underTektonDir(rel): - case hintsPipeline(rel): - data, err := os.ReadFile(p) - if err != nil { - return err - } - if tekton.LooksTekton(data) { - files[rel] = data - } - return nil - default: - return nil - } - data, err := os.ReadFile(p) + data, keep, err := selectTreeFile(rel, func() ([]byte, error) { return os.ReadFile(p) }) if err != nil { return err } - files[rel] = data + if keep { + files[rel] = data + } return nil }) if err != nil { @@ -80,6 +70,108 @@ func ReadTree(root string) (map[string][]byte, error) { return files, nil } +// Tarball streaming limits: ReadTarball keeps at most MaxTarballFileBytes +// per matched file and MaxTarballKeptBytes across all kept files, so a +// monorepo's source tree never has to fit anywhere — only its CI files do. +const ( + MaxTarballFileBytes = 1 << 20 + MaxTarballKeptBytes = 32 << 20 +) + +// ErrTarballKeptTooLarge is returned when the CI files alone exceed +// MaxTarballKeptBytes. +var ErrTarballKeptTooLarge = errors.New("ingest: pipeline files exceed size cap") + +// ReadTarball streams a gzipped tar archive (GitHub's /tarball shape: one +// wrapping top-level directory, a leading pax_global_header) and returns the +// same selection ReadTree makes from a checkout, without extracting +// anything to disk. Entries outside the selection are discarded as they +// stream, so the archive's total size is unbounded; a matched file over +// MaxTarballFileBytes is skipped (and Repository never sees it), and the +// kept set is capped at MaxTarballKeptBytes. +func ReadTarball(r io.Reader) (map[string][]byte, error) { + gz, err := gzip.NewReader(r) + if err != nil { + return nil, fmt.Errorf("ingest: tarball gzip: %w", err) + } + defer gz.Close() + tr := tar.NewReader(gz) + files := map[string][]byte{} + var kept int64 + for { + hdr, err := tr.Next() + if err == io.EOF { + return files, nil + } + if err != nil { + return nil, fmt.Errorf("ingest: tarball: %w", err) + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + continue + } + rel, ok := tarballRelPath(hdr.Name) + if !ok || len(files) >= MaxTreeFiles || hdr.Size > MaxTarballFileBytes { + continue + } + data, keep, err := selectTreeFile(rel, func() ([]byte, error) { + return io.ReadAll(io.LimitReader(tr, MaxTarballFileBytes+1)) + }) + if err != nil { + return nil, fmt.Errorf("ingest: tarball %s: %w", rel, err) + } + if !keep || int64(len(data)) > MaxTarballFileBytes { + continue + } + kept += int64(len(data)) + if kept > MaxTarballKeptBytes { + return nil, ErrTarballKeptTooLarge + } + files[rel] = data + } +} + +// tarballRelPath strips the wrapping top-level directory from a GitHub +// tarball entry and rejects anything under a skipDirs component or that +// escapes the tree. +func tarballRelPath(name string) (string, bool) { + name = strings.TrimPrefix(filepath.ToSlash(name), "./") + _, rel, ok := strings.Cut(name, "/") + if !ok || rel == "" || strings.HasSuffix(rel, "/") { + return "", false + } + for _, seg := range strings.Split(rel, "/") { + if seg == ".." || skipDirs[seg] { + return "", false + } + } + return rel, true +} + +// selectTreeFile applies the ReadTree selection to one repo-relative path, +// calling read only when the path is a candidate; keep reports whether +// the returned data belongs in the result. +func selectTreeFile(rel string, read func() ([]byte, error)) (data []byte, keep bool, err error) { + switch { + case IsIntakePath(rel): + case !isYAMLPath(rel): + return nil, false, nil + case githubactions.IsWorkflowPath(rel), underTektonDir(rel): + case hintsPipeline(rel): + data, err := read() + if err != nil { + return nil, false, err + } + return data, tekton.LooksTekton(data), nil + default: + return nil, false, nil + } + data, err = read() + if err != nil { + return nil, false, err + } + return data, true, nil +} + func underTektonDir(rel string) bool { for _, d := range tektonDirs { if strings.HasPrefix(rel, d) { diff --git a/src/pkg/workflowmodel/ingest/tree_test.go b/src/pkg/workflowmodel/ingest/tree_test.go index 6c25e1b936..3ee53d627e 100644 --- a/src/pkg/workflowmodel/ingest/tree_test.go +++ b/src/pkg/workflowmodel/ingest/tree_test.go @@ -1,6 +1,9 @@ package ingest import ( + "archive/tar" + "bytes" + "compress/gzip" "os" "path/filepath" "sort" @@ -64,3 +67,69 @@ func TestReadTreeFeedsLifecycle(t *testing.T) { t.Fatalf("pipelines %d", len(m.Spec.Pipelines)) } } + +func TestReadTarballStreamsSelection(t *testing.T) { + type entry struct { + name string + body []byte + flag byte + } + big := bytes.Repeat([]byte("x"), 3*MaxTarballFileBytes) + entries := []entry{ + {name: "pax_global_header", flag: tar.TypeXGlobalHeader}, + {name: "acme-widgets-abc123/", flag: tar.TypeDir}, + {name: "acme-widgets-abc123/.github/workflows/ci.yml", body: []byte("on: push\njobs: {}\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/.tekton/pr.yaml", body: []byte("apiVersion: tekton.dev/v1\nkind: PipelineRun\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/deploy/release-pipeline.yaml", body: []byte("apiVersion: tekton.dev/v1\nkind: Pipeline\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/deploy/other-pipeline.yaml", body: []byte("kind: Deployment\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/vendor/x/.github/workflows/v.yml", body: []byte("on: push\n"), flag: tar.TypeReg}, + // Source blobs far past any per-file cap stream through untouched. + {name: "acme-widgets-abc123/third_party/blob.bin", body: big, flag: tar.TypeReg}, + {name: "acme-widgets-abc123/assets/huge.tar", body: big, flag: tar.TypeReg}, + // A matched path over the per-file cap is skipped, not fatal. + {name: "acme-widgets-abc123/.github/workflows/huge.yml", body: big, flag: tar.TypeReg}, + {name: "acme-widgets-abc123/README.md", body: []byte("hi\n"), flag: tar.TypeReg}, + } + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, e := range entries { + hdr := &tar.Header{Name: e.name, Typeflag: e.flag, Mode: 0o644, Size: int64(len(e.body))} + if e.flag == tar.TypeXGlobalHeader { + hdr = &tar.Header{Name: e.name, Typeflag: e.flag, PAXRecords: map[string]string{"comment": "abc123"}} + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(e.body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + + files, err := ReadTarball(&buf) + if err != nil { + t.Fatal(err) + } + var got []string + for p := range files { + got = append(got, p) + } + sort.Strings(got) + want := ".github/workflows/ci.yml .tekton/pr.yaml deploy/release-pipeline.yaml" + if strings.Join(got, " ") != want { + t.Fatalf("got %v", got) + } + if string(files[".tekton/pr.yaml"]) != "apiVersion: tekton.dev/v1\nkind: PipelineRun\n" { + t.Fatalf("content: %q", files[".tekton/pr.yaml"]) + } + + if _, err := ReadTarball(strings.NewReader("not gzip")); err == nil { + t.Fatal("expected gzip error") + } +} From b3f5ab48c3f32d90663116d49e41dabb83c19742 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:21:54 +0000 Subject: [PATCH 020/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20redact=20proxy=20?= =?UTF-8?q?userinfo=20in=20upstreamproxy=20URL=20errors=20(#17731)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: redact proxy userinfo in upstreamproxy URL errors parseProxyURL formatted the raw proxy URL into its rejection errors, and configureUpstreamProxy logs that error, so a malformed URL carrying the proxy's Basic credential put the password in the hive JSON log on every boot. logscrub's basic-auth category covers Authorization headers only, not URL userinfo, so nothing downstream masked it. Use url.Redacted() on the parsed form, strip the userinfo segment when url.Parse itself fails, and drop the *url.Error wrapper whose message repeats the full URL. Fixes #17730 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * chore: re-key merge request after transient lint-pending refusal Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...security-17730-upstreamproxy-url-redact.md | 1 + src/crux/upstreamproxy/upstreamproxy.go | 28 +++++++++++++++++-- src/crux/upstreamproxy/upstreamproxy_test.go | 21 ++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 changelog.d/security-17730-upstreamproxy-url-redact.md diff --git a/changelog.d/security-17730-upstreamproxy-url-redact.md b/changelog.d/security-17730-upstreamproxy-url-redact.md new file mode 100644 index 0000000000..432e2a58ef --- /dev/null +++ b/changelog.d/security-17730-upstreamproxy-url-redact.md @@ -0,0 +1 @@ +- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). diff --git a/src/crux/upstreamproxy/upstreamproxy.go b/src/crux/upstreamproxy/upstreamproxy.go index 723dfa230b..52abb11359 100644 --- a/src/crux/upstreamproxy/upstreamproxy.go +++ b/src/crux/upstreamproxy/upstreamproxy.go @@ -144,17 +144,24 @@ func parseProxyURL(raw string) (*url.URL, error) { if !strings.Contains(raw, "://") { raw = "http://" + raw } + // Errors are logged by the caller; the URL's userinfo is the proxy's + // Basic credential, so never echo raw. u, err := url.Parse(raw) if err != nil { - return nil, fmt.Errorf("upstream proxy url %q: %w", raw, err) + // *url.Error.Error() repeats the full URL; keep only its inner cause. + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return nil, fmt.Errorf("upstream proxy url %q: %w", redactUserinfo(raw), err) } switch u.Scheme { case "http", "https": default: - return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", raw, u.Scheme) + return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", u.Redacted(), u.Scheme) } if u.Hostname() == "" { - return nil, fmt.Errorf("upstream proxy url %q: missing host", raw) + return nil, fmt.Errorf("upstream proxy url %q: missing host", u.Redacted()) } if u.Port() == "" { port := "80" @@ -166,6 +173,21 @@ func parseProxyURL(raw string) (*url.URL, error) { return u, nil } +// redactUserinfo masks everything between "://" and the first "@" of raw, for +// URLs url.Parse rejected and so cannot be redacted structurally. +func redactUserinfo(raw string) string { + i := strings.Index(raw, "://") + if i < 0 { + return raw + } + rest := raw[i+3:] + at := strings.Index(rest, "@") + if at < 0 { + return raw + } + return raw[:i+3] + "xxxxx@" + rest[at+1:] +} + func envList(getenv func(string) string, name string) ([]string, bool) { raw := getenv(name) if strings.TrimSpace(raw) == "" { diff --git a/src/crux/upstreamproxy/upstreamproxy_test.go b/src/crux/upstreamproxy/upstreamproxy_test.go index f3d45b8623..5c7d84dcab 100644 --- a/src/crux/upstreamproxy/upstreamproxy_test.go +++ b/src/crux/upstreamproxy/upstreamproxy_test.go @@ -144,6 +144,27 @@ func TestResolveRejectsBadInput(t *testing.T) { } } +func TestResolveErrorsRedactUserinfo(t *testing.T) { + // The caller logs the error; the URL's userinfo is the proxy's Basic + // credential and must not reach the log through any rejection branch. + for _, raw := range []string{ + "socks5://alice:s3cret-pw@proxy.corp:1080", // unsupported scheme + "http://alice:s3cret-pw@", // missing host + "http://alice:s3cret-pw@proxy.corp:%zz", // url.Parse failure + } { + _, err := Resolve(Config{URL: raw}, nil) + if err == nil { + t.Fatalf("%q: accepted", raw) + } + if strings.Contains(err.Error(), "s3cret-pw") { + t.Fatalf("%q: error leaks password: %v", raw, err) + } + if !strings.Contains(err.Error(), "alice") && !strings.Contains(err.Error(), "xxxxx") { + t.Fatalf("%q: error lost the redacted URL entirely: %v", raw, err) + } + } +} + func TestGitEnv(t *testing.T) { sel, _ := Resolve(Config{URL: "http://p:3128", Hosts: []string{"github.ibm.com", "*.github.ibm.com"}, NoProxy: []string{"github.com"}}, nil) got := strings.Join(sel.GitEnv(), "\n") From ed0d01a2d4eb6d0099e91c64d58b61d271e05c33 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:22:05 -0700 Subject: [PATCH 021/223] =?UTF-8?q?=E2=9C=A8=20benchmarks:=20per-PR=20esti?= =?UTF-8?q?mated=20cost=20in=20/api/benchmarks/export=20(#17791)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records now carry cost.basis (tokens_x_pricing | unknown), cost.usd_est, token share, launches and price_table_date, attributed from the spoke's launch-cost collector: the swarm's PR ref for reviewer records, the governor's issue ref of an issue the PR closed for author records (both ref spellings). Vendored benchmarks schema refreshed; converter and dashboard lookup tests; docs cost section. Fixes #17790 Refs #17694 Signed-off-by: Andrew Anderson --- changelog.d/added-benchmarks-export-cost.md | 1 + src/docs/benchmarks.md | 22 ++++- src/pkg/dashboard/api_benchmarks.go | 1 + src/pkg/dashboard/benchmarks_cost.go | 80 +++++++++++++++++++ src/pkg/dashboard/benchmarks_cost_test.go | 43 ++++++++++ src/pkg/evidence/benchmarks/convert.go | 48 +++++++++++ src/pkg/evidence/benchmarks/convert_test.go | 67 +++++++++++++++- src/pkg/evidence/benchmarks/record.go | 35 ++++++++ .../benchmarks/testdata/result.schema.json | 46 +++++++---- 9 files changed, 323 insertions(+), 20 deletions(-) create mode 100644 changelog.d/added-benchmarks-export-cost.md create mode 100644 src/pkg/dashboard/benchmarks_cost.go create mode 100644 src/pkg/dashboard/benchmarks_cost_test.go diff --git a/changelog.d/added-benchmarks-export-cost.md b/changelog.d/added-benchmarks-export-cost.md new file mode 100644 index 0000000000..62f95173f7 --- /dev/null +++ b/changelog.d/added-benchmarks-export-cost.md @@ -0,0 +1 @@ +- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md index be9cf68661..ef2ef3c072 100644 --- a/src/docs/benchmarks.md +++ b/src/docs/benchmarks.md @@ -28,11 +28,28 @@ One record per PR per role, conforming to | `task.acmm_level` | `policy.acmm_level` at review time. | | `outcome.*` | `pr_opened`, `pr_merged`, `merged_at`, `time_to_merge_s`, `ci_status_at_merge`, `ci_green_first_try`, `review_verdict`, `reverted`, `closed_unmerged`, `human_interventions` (human approvals / label changes). | | `quality.*` | Swarm verdict counts and confidence. | +| `cost.*` | Estimated spend — see [Cost](#cost-estimated-never-billed). | **What never leaves the spoke:** the hive id, repository and organization names (unless the repository is public — see below), PR numbers, commit SHAs, -agent prompts, diff content, reviewer comments, human logins, and cost/token -figures (not present in bundles today). +agent prompts, diff content, reviewer comments, human logins, per-session +token usage and any billed figure. Only the per-PR *estimated* USD and its +token share leave, rounded by the price table. + +## Cost: estimated, never billed + +Each record carries `cost.basis` and, when attributable, `cost.usd_est`: + +| `cost.basis` | Meaning | +|---|---| +| `tokens_x_pricing` | The spoke's launch-cost collector (`/api/launch-cost`, `/data/launch-cost.json`) saw a kick that named this PR — the swarm's `owner/repo#N` PR ref for reviewer records, or the governor's `repo#N` issue ref of an issue the PR closed (`issue_closures`) for author records. The kick's token usage is priced with `pkg/tokens/pricing.go` (public list prices, `price_table_date`) and split evenly across the refs the kick named — the same `share_usd` the Cost panel shows. `launches` counts the kicks folded in; `*_tokens` are the in-window token share, informational. | +| `unknown` | No kick with usage named the PR: kicked without a ref, older than the launch-cost store, or usage landed outside the 6 h ownership horizon. | + +The spoke never emits `fleet_model_median`; the benchmarks site imputes that +at index-build time from attributed records of the same model and role and +labels it as such. `reported` (a gateway-billed figure) is reserved. List +prices are not what subscription plans or self-hosted inference cost, so the +site shows every dollar as an estimate with its basis mix. ## Privacy model: default on, anonymized @@ -97,5 +114,6 @@ pods and a nightly workflow rebuilds the index. - `pkg/evidence/benchmarks` — record types, `HiveHash`, `Convert` (bundles → records), schema golden test. - `pkg/dashboard/api_benchmarks.go` — export endpoint, status block, hourly pusher, Settings panel data. +- `pkg/dashboard/benchmarks_cost.go` — `CostLookup` over the launch-cost collector (daily rollup USD + snapshot tokens). - `pkg/hub/benchmarks.go` — ingest, month-file store, hourly GitHub publisher, status endpoint. - `pkg/evidence` `Author.runtime` / `Author.task_kind` — stamped at bundle write via `github.EvidenceAgentContext`. diff --git a/src/pkg/dashboard/api_benchmarks.go b/src/pkg/dashboard/api_benchmarks.go index ae05bddc3c..c11c0608c0 100644 --- a/src/pkg/dashboard/api_benchmarks.go +++ b/src/pkg/dashboard/api_benchmarks.go @@ -147,6 +147,7 @@ func (s *Server) benchmarksOptions(ctx context.Context, since time.Time) benchma RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), RepoPublic: func(repo string) bool { return s.benchmarksRepoPublic(ctx, repo) }, SandboxRuntime: cfg.BenchmarkSandboxRuntime, + CostLookup: benchmarksCostLookup(s.deps.LaunchCost), } } diff --git a/src/pkg/dashboard/benchmarks_cost.go b/src/pkg/dashboard/benchmarks_cost.go new file mode 100644 index 0000000000..650523332f --- /dev/null +++ b/src/pkg/dashboard/benchmarks_cost.go @@ -0,0 +1,80 @@ +package dashboard + +import ( + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + "github.com/hivecommons/hive/pkg/tokens" +) + +// benchmarksCostLookup attributes kick spend to a benchmark record from the +// launch-cost collector: USD from the persisted daily rollup (every day the +// store has seen, live days at their latest share) and token counts from the +// in-window snapshot, each launch split evenly across the refs it named — +// the same accounting as /api/launch-cost by_ref[].share_usd. Nil collector +// means nothing is attributed. +func benchmarksCostLookup(lc *collect.LaunchCostCollector) benchmarks.CostLookup { + if lc == nil { + return nil + } + return func(refs []benchmarks.CostRef) (benchmarks.Cost, bool) { + want := map[benchmarks.CostRef]bool{} + for _, r := range refs { + want[r] = true + } + var usd float64 + var launches int + for _, day := range lc.DailyRollup() { + for _, ref := range day.ByRef { + if want[benchmarks.CostRef{Ref: ref.Ref, Kind: ref.Kind}] { + usd += ref.ShareUSD + launches += ref.Launches + } + } + } + if usd <= 0 { + return benchmarks.Cost{Basis: benchmarks.CostBasisUnknown}, false + } + c := benchmarks.Cost{ + Basis: benchmarks.CostBasisTokensXPricing, + USDEst: &usd, + Launches: launches, + PriceTableDate: tokens.PriceTableDate(), + } + if snap, ok := lc.Snapshot(); ok { + addLaunchTokens(&c, snap.ByLaunch, want) + } + return c, true + } +} + +// addLaunchTokens adds the per-ref token share of every in-window launch +// that named one of the wanted refs. Tokens are informational: usd_est is +// already priced by the collector. +func addLaunchTokens(c *benchmarks.Cost, launches []collect.LaunchCostEntry, want map[benchmarks.CostRef]bool) { + var in, out, read, write int64 + seen := false + for _, l := range launches { + n := int64(len(l.Refs)) + if n == 0 { + continue + } + hit := 0 + for _, r := range l.Refs { + if want[benchmarks.CostRef{Ref: r.Ref, Kind: r.Kind}] { + hit++ + } + } + if hit == 0 { + continue + } + seen = true + k := int64(hit) + in += l.Input * k / n + out += l.Output * k / n + read += l.CacheRead * k / n + write += l.CacheCreate * k / n + } + if seen { + c.InputTokens, c.OutputTokens, c.CacheReadTokens, c.CacheWriteTokens = &in, &out, &read, &write + } +} diff --git a/src/pkg/dashboard/benchmarks_cost_test.go b/src/pkg/dashboard/benchmarks_cost_test.go new file mode 100644 index 0000000000..be62d6712f --- /dev/null +++ b/src/pkg/dashboard/benchmarks_cost_test.go @@ -0,0 +1,43 @@ +package dashboard + +import ( + "context" + "testing" + + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + "github.com/hivecommons/hive/pkg/tokens" +) + +func TestBenchmarksCostLookupFromLaunchCost(t *testing.T) { + if benchmarksCostLookup(nil) != nil { + t.Fatal("nil collector must disable attribution") + } + summary, launches, _ := launchFixture(t) + lc := collect.NewLaunchCostCollector(&fakeLaunchTokens{summary: summary, launches: launches}, nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + lc.Start(ctx) + + lookup := benchmarksCostLookup(lc) + // Reviewer record: PR ref as the swarm spells it. + c, ok := lookup([]benchmarks.CostRef{{Ref: "o/r#9", Kind: tokens.LaunchRefKindPR}, {Ref: "r#9", Kind: tokens.LaunchRefKindPR}}) + if !ok || c.Basis != benchmarks.CostBasisTokensXPricing || c.USDEst == nil || *c.USDEst <= 0 { + t.Fatalf("pr ref not attributed: %+v", c) + } + if c.Launches != 1 || c.PriceTableDate != tokens.PriceTableDate() { + t.Fatalf("launches/price table: %+v", c) + } + if c.InputTokens == nil || *c.InputTokens != 50 || c.OutputTokens == nil || *c.OutputTokens != 5 { + t.Fatalf("tokens from snapshot: in=%v out=%v", c.InputTokens, c.OutputTokens) + } + // Author record closing issue 1: governor ref, issue kind. + c, ok = lookup([]benchmarks.CostRef{{Ref: "o/r#1", Kind: tokens.LaunchRefKindIssue}}) + if !ok || *c.USDEst <= 0 || *c.InputTokens != 200 { + t.Fatalf("issue ref not attributed: %+v", c) + } + // Unknown ref: nothing landed. + if c, ok = lookup([]benchmarks.CostRef{{Ref: "o/r#99", Kind: tokens.LaunchRefKindIssue}}); ok || c.Basis != benchmarks.CostBasisUnknown { + t.Fatalf("unrelated ref must be unknown: %+v", c) + } +} diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go index 0f9f6c5966..6f52f14ab5 100644 --- a/src/pkg/evidence/benchmarks/convert.go +++ b/src/pkg/evidence/benchmarks/convert.go @@ -27,6 +27,9 @@ type Options struct { // SandboxRuntime resolves the runtime an agent's kicks ran under // ("none" when sandboxing is off). nil means "none" for every agent. SandboxRuntime func(agent string) string + // CostLookup attributes kick spend to a PR (author: the PR and the issues + // it closed; reviewer: the PR). nil leaves cost.basis "unknown". + CostLookup CostLookup } // Roles. @@ -133,6 +136,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)} rec.SandboxRuntime = runtimeOf(opts, a, name) rec.Task.Kind = kindOf(a.TaskKind, name) + rec.Cost = costOf(opts.CostLookup, authorCostRefs(latest)) out = append(out, rec) } if backend, model, ok := dominantReviewer(latest.Verdicts); ok { @@ -142,6 +146,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec.Model = Model{Provider: providerOf(model), Name: model} rec.SandboxRuntime = SandboxRuntimeNone rec.Task.Kind = kindPRReview + rec.Cost = costOf(opts.CostLookup, prCostRefs(latest.Repo, latest.Number, costKindPR)) out = append(out, rec) } } @@ -149,6 +154,49 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { return out } +const ( + costKindIssue = "issue" + costKindPR = "pr" +) + +// prCostRefs spells one ref both ways the kick path records it. +func prCostRefs(repo string, number int, kind string) []CostRef { + full := repo + "#" + strconv.Itoa(number) + refs := []CostRef{{Ref: full, Kind: kind}} + if _, short, ok := strings.Cut(repo, "/"); ok && short != "" { + refs = append(refs, CostRef{Ref: short + "#" + strconv.Itoa(number), Kind: kind}) + } + return refs +} + +// authorCostRefs lists the refs an authoring kick may have been launched +// against: the PR itself and every same-repo issue it closed. +func authorCostRefs(b *evidence.Bundle) []CostRef { + refs := prCostRefs(b.Repo, b.Number, costKindIssue) + for _, c := range b.IssueClosures { + repo := c.Repo + if repo == "" { + repo = b.Repo + } + if c.Issue > 0 { + refs = append(refs, prCostRefs(repo, c.Issue, costKindIssue)...) + } + } + return refs +} + +func costOf(lookup CostLookup, refs []CostRef) Cost { + if lookup != nil { + if c, ok := lookup(refs); ok && c.USDEst != nil && *c.USDEst > 0 { + if c.Basis == "" { + c.Basis = CostBasisTokensXPricing + } + return c + } + } + return Cost{Basis: CostBasisUnknown} +} + // maxACMM is the highest ACMM level the schema accepts. const maxACMM = 6 diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go index b70a87d042..6bef3b93b8 100644 --- a/src/pkg/evidence/benchmarks/convert_test.go +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -28,9 +28,10 @@ func fixtureBundles() []*evidence.Bundle { {Perspective: "security", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "approve", Confidence: 0.9}, {Perspective: "style", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "comment", Confidence: 0.7}, }, - HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, - Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, - CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), + HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, + IssueClosures: []evidence.IssueClosure{{Repo: "hivecommons/hive", Issue: 40, At: t0(3 * time.Hour)}}, + Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, + CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), } private := &evidence.Bundle{ SchemaVersion: evidence.SchemaVersion, ID: "c", Repo: "acme/secret", Number: 7, @@ -53,6 +54,66 @@ func testOptions() Options { RepoConsent: true, RepoPublic: func(repo string) bool { return strings.HasPrefix(repo, "hivecommons/") }, SandboxRuntime: func(agent string) string { return "none" }, + CostLookup: fixtureCostLookup, + } +} + +// fixtureCostLookup mimics the launch-cost store: the governor kicked short +// "hive#40" (issue the PR closed), the swarm reviewed full "hivecommons/hive#42". +func fixtureCostLookup(refs []CostRef) (Cost, bool) { + spend := map[CostRef]float64{ + {Ref: "hive#40", Kind: "issue"}: 1.25, + {Ref: "hivecommons/hive#42", Kind: "pr"}: 0.40, + {Ref: "acme/secret#7", Kind: "issue"}: 0, // kick landed no usage + } + var usd float64 + n := 0 + for _, r := range refs { + if v, ok := spend[r]; ok { + usd += v + n++ + } + } + if n == 0 { + return Cost{Basis: CostBasisUnknown}, false + } + return Cost{Basis: CostBasisTokensXPricing, USDEst: &usd, Launches: n, PriceTableDate: "2026-11-09"}, true +} + +func TestConvertAttributesCost(t *testing.T) { + recs := Convert(fixtureBundles(), testOptions()) + author := byRole(recs, "hivecommons/hive", RoleAuthor) + if author == nil || author.Cost.Basis != CostBasisTokensXPricing || author.Cost.USDEst == nil || *author.Cost.USDEst != 1.25 { + t.Fatalf("author cost via closed issue short ref: %+v", author) + } + reviewer := byRole(recs, "hivecommons/hive", RoleReviewer) + if reviewer == nil || reviewer.Cost.USDEst == nil || *reviewer.Cost.USDEst != 0.40 || reviewer.Cost.Launches != 1 { + t.Fatalf("reviewer cost via full pr ref: %+v", reviewer) + } + private := byRole(recs, "", RoleAuthor) + if private == nil || private.Cost.Basis != CostBasisUnknown || private.Cost.USDEst != nil { + t.Fatalf("zero-usage kick must be unknown: %+v", private) + } + noLookup := testOptions() + noLookup.CostLookup = nil + for _, r := range Convert(fixtureBundles(), noLookup) { + if r.Cost.Basis != CostBasisUnknown { + t.Fatalf("nil lookup must yield unknown, got %+v", r.Cost) + } + } +} + +func TestCostRefsSpellBothForms(t *testing.T) { + b := &evidence.Bundle{Repo: "o/r", Number: 5, IssueClosures: []evidence.IssueClosure{{Issue: 3}}} + got := authorCostRefs(b) + want := []CostRef{{"o/r#5", "issue"}, {"r#5", "issue"}, {"o/r#3", "issue"}, {"r#3", "issue"}} + if len(got) != len(want) { + t.Fatalf("refs %v", got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("ref %d = %v want %v", i, got[i], want[i]) + } } } diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go index 50b2aaf0f9..875297febd 100644 --- a/src/pkg/evidence/benchmarks/record.go +++ b/src/pkg/evidence/benchmarks/record.go @@ -49,6 +49,7 @@ type Record struct { Task Task `json:"task"` Outcome Outcome `json:"outcome"` Quality *Quality `json:"quality,omitempty"` + Cost Cost `json:"cost"` } // Source says which hive emitted the record. @@ -109,6 +110,40 @@ type Quality struct { SentinelFindings *int `json:"sentinel_findings,omitempty"` } +// Cost basis values (schema cost.basis). A spoke never emits +// fleet_model_median — the index builder imputes that from published records. +const ( + CostBasisReported = "reported" + CostBasisTokensXPricing = "tokens_x_pricing" + CostBasisUnknown = "unknown" +) + +// Cost is the record's estimated spend. USDEst is hive's launch attribution +// priced with its public list-price table; it is an estimate, never a bill. +type Cost struct { + Basis string `json:"basis"` + USDEst *float64 `json:"usd_est,omitempty"` + InputTokens *int64 `json:"input_tokens,omitempty"` + OutputTokens *int64 `json:"output_tokens,omitempty"` + CacheReadTokens *int64 `json:"cache_read_tokens,omitempty"` + CacheWriteTokens *int64 `json:"cache_write_tokens,omitempty"` + // Launches counts the kicks whose spend was attributed to the record. + Launches int `json:"launches,omitempty"` + PriceTableDate string `json:"price_table_date,omitempty"` +} + +// CostRef is one work reference a kick may have named. Ref carries the +// spelling the kick path uses: issue refs are "repo#N" (governor), PR refs +// are "owner/repo#N" (review swarm); Convert asks for both. +type CostRef struct { + Ref string + Kind string // "issue" or "pr" +} + +// CostLookup sums the attributed spend of every ref and reports whether any +// usage landed. Spend of zero counts as no attribution. +type CostLookup func(refs []CostRef) (Cost, bool) + // recordID derives the stable id of one (hive, PR, role) triple so re-exports // overwrite rather than duplicate. func recordID(hiveHash, repo string, number int, role string) string { diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json index c4ecd39ec6..87f9edd6bb 100644 --- a/src/pkg/evidence/benchmarks/testdata/result.schema.json +++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json @@ -357,7 +357,28 @@ "cost": { "type": "object", "additionalProperties": false, + "description": "Estimated spend for this record. usd_est is ALWAYS an estimate (list prices, apportioned kicks); basis says how it was derived. Records with basis=unknown carry no usd_est and are excluded from cost aggregates.", "properties": { + "usd_est": { + "type": "number", + "minimum": 0, + "description": "Estimated USD for this PR/review." + }, + "basis": { + "type": "string", + "enum": [ + "reported", + "tokens_x_pricing", + "fleet_model_median", + "unknown" + ], + "description": "reported: the harness/gateway returned a billed figure. tokens_x_pricing: hive's per-kick token counts \u00d7 schema/pricing.json list prices, apportioned to this PR via launch attribution (kick refs \u2192 issue_closures/PR). fleet_model_median: no attribution for this PR; median tokens_x_pricing cost for the same (model, role) across the fleet. unknown: nothing available." + }, + "usd": { + "type": "number", + "minimum": 0, + "description": "Billed USD when basis=reported." + }, "input_tokens": { "type": "integer", "minimum": 0 @@ -374,25 +395,20 @@ "type": "integer", "minimum": 0 }, - "usd": { - "type": "number", - "minimum": 0 - }, - "basis": { - "type": "string", - "description": "How usd was computed (GET /api/cost: estimated from public price table, or apportioned from per-agent totals).", - "enum": [ - "estimated", - "apportioned", - "billed", - "unpriced" - ] + "launches": { + "type": "integer", + "minimum": 0, + "description": "Agent kicks attributed to this PR (tokens_x_pricing only)." }, "price_table_date": { "type": "string", - "format": "date" + "format": "date", + "description": "as_of of the price table used." } - } + }, + "required": [ + "basis" + ] }, "quality": { "type": "object", From d8dc1ce781ace4b7f2ec716dec0602308402477c Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 23:46:33 +0000 Subject: [PATCH 022/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.223.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.223.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 13 +++++++++++++ changelog.d/added-substrate-preflight-prover.md | 1 - changelog.d/fixed-marketplace-seed-resume.md | 1 - changelog.d/fixed-openshell-sandbox-name.md | 1 - .../fixed-prow-oktotest-rejection-warn-once.md | 1 - .../fixed-status-snapshot-bounded-staleness.md | 1 - 6 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-substrate-preflight-prover.md delete mode 100644 changelog.d/fixed-marketplace-seed-resume.md delete mode 100644 changelog.d/fixed-openshell-sandbox-name.md delete mode 100644 changelog.d/fixed-prow-oktotest-rejection-warn-once.md delete mode 100644 changelog.d/fixed-status-snapshot-bounded-staleness.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b3aa3ce8c..541d4ef742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.223.0) + +### Added + +- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. + +### Fixed + +- Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. +- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) +- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). +- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). + ## 2026-11-11 (v5.222.1) ### Fixed diff --git a/changelog.d/added-substrate-preflight-prover.md b/changelog.d/added-substrate-preflight-prover.md deleted file mode 100644 index dd9d5bd5d9..0000000000 --- a/changelog.d/added-substrate-preflight-prover.md +++ /dev/null @@ -1 +0,0 @@ -- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. diff --git a/changelog.d/fixed-marketplace-seed-resume.md b/changelog.d/fixed-marketplace-seed-resume.md deleted file mode 100644 index 5321431ab7..0000000000 --- a/changelog.d/fixed-marketplace-seed-resume.md +++ /dev/null @@ -1 +0,0 @@ -- Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. diff --git a/changelog.d/fixed-openshell-sandbox-name.md b/changelog.d/fixed-openshell-sandbox-name.md deleted file mode 100644 index 6f902b3cbd..0000000000 --- a/changelog.d/fixed-openshell-sandbox-name.md +++ /dev/null @@ -1 +0,0 @@ -- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) diff --git a/changelog.d/fixed-prow-oktotest-rejection-warn-once.md b/changelog.d/fixed-prow-oktotest-rejection-warn-once.md deleted file mode 100644 index 6e33a4e8bb..0000000000 --- a/changelog.d/fixed-prow-oktotest-rejection-warn-once.md +++ /dev/null @@ -1 +0,0 @@ -- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). diff --git a/changelog.d/fixed-status-snapshot-bounded-staleness.md b/changelog.d/fixed-status-snapshot-bounded-staleness.md deleted file mode 100644 index 30daf850a4..0000000000 --- a/changelog.d/fixed-status-snapshot-bounded-staleness.md +++ /dev/null @@ -1 +0,0 @@ -- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). From 2f53b5004ceafedf8eae613cfc6a61959a6e06d6 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:47:06 -0700 Subject: [PATCH 023/223] =?UTF-8?q?=E2=9C=A8=20marketplace:=20seed=20curat?= =?UTF-8?q?ed=20models=20and=20ship=20the=20CNCF=20TechDocs=20assessment?= =?UTF-8?q?=20workflow=20(#17798)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Curated marketplace tier gets a source of truth in the binary: pkg/hub/marketplace/curated/*.yaml is embedded, strictly decoded, checked for the curated invariants (provenance curated, a version, nothing left inferred), compiled through bundle.Export and published at tier curated on hub startup and at the start of every refresh run when that name@version is absent. Bad files are logged and skipped, never fatal; edits without a version bump are refused as conflicts, so a new version is a new file. Seeded keys are reported in lastRefresh.curated. The first curated model is cncf-techdocs-assessment@curated-2026.11.1: Nate W.'s AI-assisted TechDocs assessment process (spec at nate-double-u/cncf-techdocs@882be6f, intake template @ad91530), scoped to cncf/techdocs. Issues travel requested → accepted → draft-open (→ escalated) → closed/declined; deliverable PRs travel drafting → in-review → stakeholder-review → awaiting-merge → merged with checks-failing and discarded branches. Every gate (/accept, /decline, /verify, /ready, /confirm, /approve, /discard, /abort) is released by a human role; the drafter and verifier agents stop at gates. Checks are the repo's format/link/spell-check workflows plus the manual provenance block. The rules evaluator is exercised over hand-written Facts for each step. cncf/techdocs is not a landscape project, so the corpus package gains an operator-added extra.json merged into Repos(); the ingested-unreviewed version now sits next to the curated one. Docs: marketplace section of docs/design/workflow-models.md (curated seeding, where a model goes, how versions work) and docs/workflows/techdocs-assessment.md for the TechDocs team. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../added-marketplace-curated-techdocs.md | 1 + src/docs/design/workflow-models.md | 32 +- src/docs/workflows/techdocs-assessment.md | 148 ++++ .../curated/cncf-techdocs-assessment.yaml | 634 ++++++++++++++++++ src/pkg/hub/marketplace/curated/curated.go | 96 +++ .../hub/marketplace/curated/curated_test.go | 212 ++++++ src/pkg/hub/marketplace/curated/load_test.go | 46 ++ src/pkg/hub/marketplace_curated.go | 56 ++ src/pkg/hub/marketplace_curated_test.go | 159 +++++ src/pkg/hub/marketplace_refresh.go | 22 +- src/pkg/hub/marketplace_test.go | 29 +- src/pkg/workflowmodel/ingest/corpus/corpus.go | 83 ++- .../ingest/corpus/corpus_test.go | 48 +- .../workflowmodel/ingest/corpus/extra.json | 8 + 14 files changed, 1534 insertions(+), 40 deletions(-) create mode 100644 changelog.d/added-marketplace-curated-techdocs.md create mode 100644 src/docs/workflows/techdocs-assessment.md create mode 100644 src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml create mode 100644 src/pkg/hub/marketplace/curated/curated.go create mode 100644 src/pkg/hub/marketplace/curated/curated_test.go create mode 100644 src/pkg/hub/marketplace/curated/load_test.go create mode 100644 src/pkg/hub/marketplace_curated.go create mode 100644 src/pkg/hub/marketplace_curated_test.go create mode 100644 src/pkg/workflowmodel/ingest/corpus/extra.json diff --git a/changelog.d/added-marketplace-curated-techdocs.md b/changelog.d/added-marketplace-curated-techdocs.md new file mode 100644 index 0000000000..4a22b467ee --- /dev/null +++ b/changelog.d/added-marketplace-curated-techdocs.md @@ -0,0 +1 @@ +- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index f025f9276b..08d1e7f5cd 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -744,9 +744,34 @@ assert `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`. `curated` and `built-in` require a hub-admin session with same-origin CSRF evidence. The hub never stores a bundle it did not itself `Verify`. +**Curated seeding.** Reviewed models ship with the hub binary under +`pkg/hub/marketplace/curated/*.yaml` (`go:embed`; loader `curated.Load`). On startup — +before any ingest, and without a GitHub token — and again at the start of every refresh +run, `seedCuratedModels` decodes each file strictly (unknown fields refused), checks the +curated invariants (`spec.meta.provenance: curated`, a `spec.meta.version`, zero +`inferred` elements), compiles it through `bundle.Export`, sets `Tier=curated` and +`Publish`es it when that `name@version` is absent. A file that fails any step is logged +(`marketplace: curated model refused` / `does not compile` / `not published`) and +skipped; the rest of the catalogue still seeds and the error appears in the run's +`errors` as `curated: …`. Seeded keys are reported in `lastRefresh.curated`, not +`published`. Entries are immutable: re-running is a no-op, and editing a file without +bumping its version is a `409` conflict at seed time, so **a new version is a new file** +(`cncf-techdocs-assessment.yaml` carries `curated-2026.11.1`; the next revision is a +second file with `curated-2026.12.1` or similar — the catalogue keeps both and the spoke +pins what it imported). To add a curated model: write the YAML (start from +`ingest.Lifecycle` + `spec.Ingest` output, confirm every inferred element, cite the +source document in `workflow.hivecommons.dev/source*` annotations and the element +`citations`), drop it in that directory, and add rules-evaluator cases to +`curated_test.go` — `TestLoadEmbedded` already refuses anything that does not compile +clean. The first curated model is the CNCF TechDocs assessment, +[docs/workflows/techdocs-assessment.md](../workflows/techdocs-assessment.md). + **Refresh.** The corpus is the CNCF survey list, now embedded as -`pkg/workflowmodel/ingest/corpus` (262 projects / 299 repositories; `cmd/workflow-survey` -falls back to it when no `projects.json` is given) plus +`pkg/workflowmodel/ingest/corpus` (`corpus.json`, 262 projects / 299 repositories; +`cmd/workflow-survey` falls back to it when no `projects.json` is given) plus the +operator-added non-landscape list `extra.json` in the same package (`{"name", +"category", "repos", "why"}` per entry; `cncf/techdocs` is there so its +`ingested-unreviewed` version exists next to the curated one) plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch tarball through its GitHub identity and streams it (`ingest.ReadTarball`): only the paths `ingest.ReadTree` would select are kept (per file ≤ 1 MiB, ≤ 32 MiB in total), everything @@ -831,4 +856,5 @@ served cached data. - Hub-side marketplace settings are environment variables, not hub YAML. - The hub logs publishes/refreshes through its structured logger; it has no audit store. - Ingest versions are per-day sequence numbers, not semantic; curation (promoting an ingest - to `curated`) is a manual admin publish of a reviewed model. + to `curated`) is a manual admin publish of a reviewed model or a new file under + `pkg/hub/marketplace/curated/`. diff --git a/src/docs/workflows/techdocs-assessment.md b/src/docs/workflows/techdocs-assessment.md new file mode 100644 index 0000000000..a849b775eb --- /dev/null +++ b/src/docs/workflows/techdocs-assessment.md @@ -0,0 +1,148 @@ +# CNCF TechDocs AI-assisted assessment — curated workflow model + +`cncf-techdocs-assessment@curated-2026.11.1` is the first curated model in the hub +marketplace (https://hive.hivecommons.dev → Settings → Workflows → Marketplace on any +hive). It encodes the CNCF TechDocs team's AI-assisted documentation assessment process +so a hive bound to `cncf/techdocs` runs intake, drafting, verification, stakeholder review +and approval as a visible lifecycle, with every agent action followed by a human gate. + +Source of truth (the model cites both by SHA in its `workflow.hivecommons.dev/source*` +annotations and in each element's `citations`): + +- Specification: [`specs/ai-assisted-techdocs-assessment-spec.md`](https://github.com/nate-double-u/cncf-techdocs/blob/ai-assisted-assessment-spec/specs/ai-assisted-techdocs-assessment-spec.md) + at `nate-double-u/cncf-techdocs@882be6fef28794433bc3a6a34ac1401391c1d3e4`. +- Intake template: [`.github/ISSUE_TEMPLATE/assessment-request.yml`](https://github.com/nate-double-u/cncf-techdocs/blob/ad915303f9774b9aa6b7646969897fada7975a00/.github/ISSUE_TEMPLATE/assessment-request.yml) + at `nate-double-u/cncf-techdocs@ad915303f9774b9aa6b7646969897fada7975a00`. +- Checks and labels: the `cncf/techdocs` `.github` tree (format, link and spell-check + workflows; `settings.yml` labels), as snapshotted in + `pkg/workflowmodel/ingest/testdata/techdocs/`. + +The model's scope is `github.com cncf/techdocs` — the upstream repository where the +process runs; the fork above is where the spec was written. Model file: +`pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml` +(`apiVersion: workflow.hivecommons.dev/v1alpha1`, `derivation.mode: rules`, +`meta.provenance: curated`). The same repository is also in the ingest corpus +(`pkg/workflowmodel/ingest/corpus/extra.json`), so an `ingested-unreviewed` version built +from the pipeline files alone sits next to this one in the catalogue. + +## How the spec maps onto the model + +The spec (§5 Lifecycle, §6 Phases, §12 Lifecycle bindings) describes one six-step loop — +request → accept → draft → review → stakeholder review → merge — run once per phase +(A analysis, B implementation plan, C issue backlog). The model encodes the loop once and +carries the phase as a label (`phase/assessment`, `phase/implementation`, +`phase/backlog`; `skip-implementation` makes the A merge open C). Each phase's tracking +issue and the intake issue itself travel the issue states; each AI-drafted deliverable is +a pull request that travels the PR states. + +### Columns + +| Column | States | Spec | +|---|---|---| +| Requested | `requested` | §5 step 1, §12 Request | +| Accepted | `accepted`, `escalated` | §5 step 2, §12 Accept; §5 failure path | +| Drafting | `draft-open` (issue), `drafting` (PR) | §5 step 3, §12 Draft | +| Review | `in-review`, `checks-failing` | §5 step 4, §12 Review | +| Stakeholders | `stakeholder-review` | §5 step 5, §12 Stakeholder review | +| Approval | `awaiting-merge` | §5 step 6, §12 Merge | +| Done (terminal) | `merged`, `closed`, `declined`, `discarded` | §12 Merge / Failure and abort | + +### States and rules (first match wins) + +Issues (intake and per-phase tracking issues): + +| State | Stage | Rule | Owner | +|---|---|---|---| +| `declined` | terminal | closed and `triage/declined` | — | +| `closed` | terminal | closed | — | +| `escalated` | 45 needs-decision | a hold label is present | administrator | +| `draft-open` | 40 covered-by-PR | `triage/accepted` and a covering PR exists | registered agent | +| `accepted` | 20 claimed | `triage/accepted` | reviewer (technical writer) | +| `requested` | 10 actionable | `needs-triage` | reviewer | + +Pull requests (deliverables carry the `Docs analysis` label; anything else abstains): + +| State | Stage | Rule | +|---|---|---| +| `merged` | terminal | merged | +| `discarded` | terminal | closed unmerged | +| `in-review` | 50 review-pending | draft and `verified` | +| `drafting` | 70 fixing | draft | +| `checks-failing` | 71 needs-author | checks red | +| `awaiting-merge` | 90 ready-for-human-merge | `confirmed` | +| `stakeholder-review` | 60 reviewed | ready and `verified` | +| `in-review` | 50 review-pending | ready but not yet `verified` | + +### Transitions + +| From | To | Event | Gate (who) | Spec | +|---|---|---|---|---| +| requested | accepted | `accepted` | `/accept` (reviewer) | §12 Accept | +| requested | declined | `declined` | `/decline` (reviewer) | §12 Accept | +| requested | closed | `closed` | — | withdrawn | +| accepted | draft-open | `draft-opened` | — (drafter opens the PR) | §12 Draft | +| accepted | escalated | `held` | — | §5 failure path | +| accepted | closed | `closed` | — | — | +| draft-open | accepted | `draft-withdrawn` | — | restart / hand-write / stop | +| draft-open | escalated | `held` | — | §5 failure path | +| draft-open | closed | `closed` | — | phase merged or aborted | +| escalated | accepted | `released` | — (administrator) | — | +| escalated | closed | `closed` | `/abort` (administrator) | §12 Failure and abort | +| drafting | in-review | `verified` | `/verify` (reviewer; needs `verified`) | §12 Review | +| in-review | stakeholder-review | `ready` | `/ready` (reviewer; needs `verified` + green checks) | §12 Stakeholder review | +| stakeholder-review | awaiting-merge | `confirmed` | `/confirm` (stakeholder) | §12 Stakeholder review | +| stakeholder-review / awaiting-merge | checks-failing | `checks-red` | — | — | +| checks-failing | stakeholder-review | `checks-green` | — | — | +| awaiting-merge | merged | `merged` | `/approve` (approver; needs `confirmed` + green) | §5 step 6, §12 Merge | +| drafting / in-review / checks-failing / stakeholder-review / awaiting-merge | discarded | `closed` | `/discard` (reviewer) | — | + +Every gate is released by a human role; the two agent profiles (`techdocs-drafter` in +`accepted`/`drafting`, `techdocs-verifier` in `drafting`) can only move work up to a gate, +never through one — the compiler's `agent-no-human-gate` refusal enforces this. + +### Roles + +| Role | Selector | Spec | +|---|---|---| +| requester | issue author (human) | §4 | +| reviewer | repository write permission (technical writer who `/accept`ed) | §4 | +| approver | repository write permission | §4 | +| stakeholder | set frozen at `/accept` from the intake's validated contacts | §4, §12 | +| administrator | repository admin | §4, §12 Failure and abort | +| drafter, verifier | the hive bot (non-human) | §4, §6 | + +### Labels + +From `settings.yml`: `Docs analysis`. Proposed for the assessment: `needs-triage`, +`triage/accepted`, `triage/declined`, `phase/assessment`, `phase/implementation`, +`phase/backlog`, `verified`, `confirmed`, `skip-implementation`. + +### Checks + +Required CI: format-check, link-check, spell-check (the three `cncf/techdocs` workflows, +ingested as pipelines with their source hashes). Required manual: provenance block present +and the strict check passes (spec §15). + +### Intake + +`assessment-request.yml` fields map to facts as recorded in the +`workflow.hivecommons.dev/intake-fields` annotation: project name, maturity and +moving-levels → assessment metadata; website/docs/repo/other-scope/out-of-scope → scope; +domains → the agent's egress allowlist; contacts → the frozen stakeholder set (validated +at `/accept`); tooling/context → drafter input; the HC-6 disclosure acknowledgement is +required. + +## Not representable today + +These parts of the spec are documented in the model's annotations but not enforced by +the engine: + +- Close *reason* (completed vs. not planned) — no fact distinguishes them. +- Approver independence from the drafter and reviewer — no fact records who drafted or + reviewed. +- Phase as a state dimension and the A→C skip — carried by labels; the + `open-next-phase-tracking-issue`, `notify-contacts` and `mention-stakeholders` + bookkeeping actions are declarative only. +- `/verify` dispatching the verifier agent and membership of the frozen stakeholder set + — the gates exist, the side effects do not yet. +- Data-collection scripts and the strict provenance check — a manual check. diff --git a/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml b/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml new file mode 100644 index 0000000000..adb197f9a2 --- /dev/null +++ b/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml @@ -0,0 +1,634 @@ +# Curated marketplace model: the CNCF TechDocs AI-assisted assessment +# lifecycle (hivecommons/hive#17620). Hand-curated from Nate W.'s written +# spec and the cncf/techdocs .github snapshot; every element is confirmed +# (no "inferred" confidence remains) and cites the spec section or file it +# came from. A new version is a new file; this one is never edited in place. +apiVersion: workflow.hivecommons.dev/v1alpha1 +kind: WorkflowModel +metadata: + name: cncf-techdocs-assessment + labels: + workflow.hivecommons.dev/tier: curated + annotations: + workflow.hivecommons.dev/source: nate-double-u/cncf-techdocs@882be6fef28794433bc3a6a34ac1401391c1d3e4 specs/ai-assisted-techdocs-assessment-spec.md + workflow.hivecommons.dev/source-url: https://github.com/nate-double-u/cncf-techdocs/blob/ai-assisted-assessment-spec/specs/ai-assisted-techdocs-assessment-spec.md + workflow.hivecommons.dev/intake-template: nate-double-u/cncf-techdocs@ad915303f9774b9aa6b7646969897fada7975a00 .github/ISSUE_TEMPLATE/assessment-request.yml + workflow.hivecommons.dev/intake-template-url: https://github.com/nate-double-u/cncf-techdocs/blob/ad915303f9774b9aa6b7646969897fada7975a00/.github/ISSUE_TEMPLATE/assessment-request.yml + # Intake form field id -> the fact or record it feeds (spec §7, §12 Accept). + workflow.hivecommons.dev/intake-fields: "project-name=assessment.project; maturity=assessment.maturity; moving-levels=assessment.criteria-level; website,docs-url,repo,other-scope,out-of-scope=assessment.scope; domains=agent.egress-allowlist; contacts=stakeholder frozen set (validated at /accept); tooling,context=drafter input; disclosure=HC-6 acknowledgement (required)" + workflow.hivecommons.dev/curated-by: Andrew Anderson + workflow.hivecommons.dev/curated-from: ingest.Lifecycle(cncf/techdocs .github snapshot) + spec.Ingest(deterministic) skeleton, then hand-curated + workflow.hivecommons.dev/spec-sections: "4. Roles; 5. Lifecycle; 6. Phases and deliverables; 7. Inputs: the assessment request; 12. Lifecycle bindings; 13. Components and repository layout; 14. Agent definitions; 15. The provenance block" +spec: + meta: + version: curated-2026.11.1 + author: CNCF TechDocs (Nate W.); curated for Hive by Andrew Anderson + description: >- + AI-assisted, expert-supervised CNCF TechDocs assessment: an intake issue is + triaged and accepted by a technical writer, an agent drafts each phase's + deliverable as a draft PR, a verifier fact-checks it, the reviewer marks it + ready, the project's stakeholders confirm factual accuracy and an + independent approver merges. Phases A/B/C repeat the same six steps and are + carried by the phase/* labels. + license: CC-BY-4.0 + source: spec + provenance: curated + scope: + forge: github.com + repository: cncf/techdocs + workTypes: [documentation-assessment] + sources: + - system: github-actions + path: .github/workflows/format-check.yml + sha256: bf8cbb5896ad92f1e88f1051998450b35d0c080b27dba24f03c5d58d54e4a67e + - system: github-actions + path: .github/workflows/link-check.yml + sha256: 1bcef2093eda4ef4ca90d5766985e7710fee8926e0e1d85e5e2055593e679e94 + - system: github-actions + path: .github/workflows/spell-check.yml + sha256: d144101194175c7382e093d176019815fbe460c3633ef892350bc2a2635d7f87 + initial: requested + columns: + - id: requested + label: Requested + hint: Intake or phase tracking issue filed; a technical writer triages it (spec §5 step 1, §12 Request) + states: [requested] + - id: accepted + label: Accepted + hint: A technical writer accepted (/accept) and owns the phase as reviewer; delegate the drafter (spec §5 step 2) + states: [accepted, escalated] + - id: drafting + label: Drafting + hint: AI-drafted deliverable as a draft PR; verifier pass runs here (spec §5 step 3, §12 Draft) + states: [draft-open, drafting] + - id: review + label: Reviewer check + hint: Verified; the reviewer refines in conversation and checks findings against source (spec §5 step 4, HC-7) + states: [in-review, checks-failing] + - id: stakeholders + label: Stakeholder review + hint: Ready for review; project stakeholders confirm factual accuracy with /confirm (spec §5 step 5, HC-2) + states: [stakeholder-review] + - id: approval + label: Approval + hint: Confirmed; an approver who was neither drafter nor reviewer signs off and merges (spec §5 step 6) + states: [awaiting-merge] + - id: done + label: Done + hint: Merged, discarded, declined or closed (spec §12 Merge, Failure and abort) + states: [merged, discarded, declined, closed] + terminal: true + states: + # ---- issues: the intake (umbrella) issue and per-phase tracking issues ---- + - id: requested + label: Requested + owner: human + origin: human + lifecycle: + column: requested + number: 10 + kinds: [issue] + waiting: [awaiting triage by a technical writer] + replies: ["/accept", "/decline"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Request. Phase A is triggered by the intake issue; the next phase becomes eligible once the previous one is merged" + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "A request issue opens labeled needs-triage ... An issue may sit unassigned indefinitely; eligibility is not work." + - path: .github/ISSUE_TEMPLATE/assessment-request.yml + line: 4 + text: "labels: Docs analysis, needs-triage" + - id: accepted + label: Accepted + owner: human + origin: human + lifecycle: + column: accepted + number: 20 + kinds: [issue] + waiting: [reviewer to delegate the phase drafter, or restart/hand-write after a discard] + replies: ["/decline"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Accept. A technical writer triages and explicitly accepts the request ... eligibility alone does not start work (P-1)" + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/accept ... assigns the issue to them, recording who accepted and who acts as the phase's reviewer, and swaps needs-triage for triage/accepted" + - path: .github/settings.yml + line: 132 + text: triage/accepted + - id: escalated + label: Escalated + owner: human + origin: human + lifecycle: + column: accepted + number: 45 + kinds: [issue] + waiting: [administrator/platform owner to abort or release the hold] + replies: ["/abort"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Repeated failure is escalated to the administrator/platform owner, who may abort the run." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "An administrator/platform owner abort closes the tracking issue and the intake with it as not planned, the rationale recorded on both" + - id: draft-open + label: Draft under way + owner: registered-agent + origin: human + lifecycle: + column: drafting + number: 40 + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "Draft (step 3): a draft pull request. The agent works on its own branch in cncf/techdocs and opens a draft PR linked to the tracking issue" + - id: declined + label: Declined + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "a writer declines an unaccepted request with /decline [reason] ... labels it triage/declined ... and closes the issue as not planned" + - path: .github/settings.yml + line: 135 + text: triage/declined + - id: closed + label: Closed + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The final phase's merge closes the intake issue as completed ... the close reason and labels say how it ended, completed at the final merge, not planned on a decline or an abort." + # ---- pull requests: the phase deliverable ---- + - id: drafting + label: Drafting + owner: registered-agent + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: drafting + number: 70 + kinds: [pr] + waiting: [drafter to finish, then the reviewer's /verify (or a recorded skip)] + replies: ["/verify", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Draft. The drafter produces the deliverable as a draft PR." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer triggers the verifier's pass by commenting /verify on the draft PR; its report lands as a PR comment ... the workflow applies the verified label" + - path: "spec:14. Agent definitions#L716-L792" + text: "Assessment drafter (Phase A) ... Plan drafter (Phase B) ... Backlog drafter (Phase C) ... Verifier (every phase)" + - id: in-review + label: Reviewer check + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: review + number: 50 + kinds: [pr] + waiting: [reviewer to verify findings against source and comment /ready] + replies: ["/ready", "/skip-implementation", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Review. The verifier's fact-check pass runs first ... The reviewer then refines the draft in conversation with the drafter, verifies findings against source (HC-7), and marks it ready." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The label is the gate: /ready refuses without it" + - path: .github/settings.yml + line: 147 + text: verified + - id: checks-failing + label: Checks failing + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: review + number: 71 + kinds: [pr] + waiting: [format, link or spelling check to go green] + replies: ["/discard"] + citations: + - path: .github/workflows/format-check.yml + text: Format checks + - path: .github/workflows/link-check.yml + text: Link checks + - path: .github/workflows/spell-check.yml + text: Spelling checks + - id: stakeholder-review + label: Stakeholder review + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: stakeholders + number: 60 + kinds: [pr] + waiting: [a stakeholder from the set frozen at acceptance to comment /confirm] + replies: ["/confirm", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Stakeholder review. Stakeholders confirm factual accuracy (required to advance) and record any disagreement with the conclusions (not required; HC-2)." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer comments /ready: the workflow marks the PR ready for review and mentions the stakeholders from the set frozen at acceptance" + - id: awaiting-merge + label: Awaiting approver + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: approval + number: 90 + kinds: [pr] + waiting: [an approver who was neither drafter nor reviewer of this phase to sign off and merge] + replies: ["/approve", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Merge. The approver signs off and merges the PR into cncf/techdocs, which makes the next phase eligible." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The approver merges, first checking the confirmation behind the confirmed label" + - path: .github/settings.yml + line: 150 + text: confirmed + - id: merged + label: Merged + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [pr] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "A merge-triggered GitHub Actions workflow then opens the next phase's tracking issue, labeled for its phase and needs-triage" + - id: discarded + label: Discarded + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [pr] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer discards a draft with /discard , which closes the PR with the reason recorded ... the tracking issue stays open for a restart or a hand-written phase." + roles: + - id: requester + label: Requester + kind: author-kind + params: {kind: human} + human: true + - id: reviewer + label: Reviewer (technical writer) + kind: permission + params: {permission: write} + human: true + - id: approver + label: Approver (independent technical writer) + kind: permission + params: {permission: write} + human: true + - id: stakeholder + label: Project stakeholder + kind: frozen-set + params: {frozenAt: accepted, source: "intake contacts validated at /accept"} + human: true + - id: administrator + label: Administrator / platform owner + kind: permission + params: {permission: admin} + human: true + - id: drafter + label: Phase drafter (agent) + kind: bot + params: {login: hive} + - id: verifier + label: Verifier (agent) + kind: bot + params: {login: hive} + gates: + - id: accept + releasedBy: reviewer + reply: "/accept" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "the workflow refuses /accept from a commenter without write access" + - id: decline + releasedBy: reviewer + reply: "/decline" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/decline [reason], under the same authorization as /accept" + - id: verified + conditions: + - {fact: labels, op: has, value: verified} + releasedBy: reviewer + reply: "/verify" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The label means the verification this PR needs is satisfied, not that the agent ran: a reviewer who deliberately skips the pass ... applies the label by hand with the reason in a comment" + - id: ready + conditions: + - {fact: labels, op: has, value: verified} + - {fact: pr.checks, op: is, value: green} + releasedBy: reviewer + reply: "/ready" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/ready refuses without it [the verified label]" + - id: confirm + conditions: + - {fact: labels, op: has, value: confirmed} + releasedBy: stakeholder + reply: "/confirm" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/confirm, which the workflow checks against the stakeholder set frozen at acceptance ... never the requester-editable intake body" + - id: approve + conditions: + - {fact: labels, op: has, value: confirmed} + - {fact: pr.checks, op: is, value: green} + releasedBy: approver + reply: "/approve" + citations: + - path: "spec:4. Roles#L166-L218" + text: "Approver. Gives each phase its independent quality sign-off and merges the PR. An experienced technical writer who was neither the drafter nor a reviewer of that phase" + - id: discard + releasedBy: reviewer + reply: "/discard" + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "the reviewer may discard it and restart or write by hand ... retries are chosen, never automatic" + - id: abort + releasedBy: administrator + reply: "/abort" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "An administrator/platform owner abort closes the tracking issue and the intake with it as not planned" + events: + - id: accepted + label: Request accepted + fact: {fact: labels, op: has, value: triage/accepted} + actor: reviewer + - id: declined + label: Request declined + fact: {fact: labels, op: has, value: triage/declined} + actor: reviewer + - id: draft-opened + label: Draft PR opened + fact: {fact: issue.covered, op: "true"} + - id: draft-withdrawn + label: Draft discarded; issue uncovered + fact: {fact: issue.covered, op: "false"} + - id: verified + label: Verification satisfied + fact: {fact: labels, op: has, value: verified} + actor: reviewer + - id: ready + label: Marked ready for review + fact: {fact: draft, op: "false"} + actor: reviewer + - id: confirmed + label: Stakeholder confirmed factual accuracy + fact: {fact: labels, op: has, value: confirmed} + actor: stakeholder + - id: checks-red + label: A check failed + fact: {fact: pr.checks, op: is, value: red} + - id: checks-green + label: Checks green + fact: {fact: pr.checks, op: is, value: green} + - id: merged + label: Deliverable merged + fact: {fact: merged, op: "true"} + actor: approver + - id: closed + label: Closed + fact: {fact: open, op: "false"} + - id: held + label: Hold applied (escalation) + fact: {fact: hold, op: "true"} + - id: released + label: Hold released + fact: {fact: hold, op: "false"} + actor: administrator + agents: + - profile: techdocs-drafter + states: [accepted, drafting] + tools: [read-repo, web-research, write-draft-pr] + produces: [pull-request, push] + - profile: techdocs-verifier + states: [drafting] + tools: [read-repo, web-research, comment] + produces: [comment] + checks: + - id: format-check + name: Format checks + kind: ci + required: true + source: .github/workflows/format-check.yml + - id: link-check + name: Link checks + kind: ci + required: true + source: .github/workflows/link-check.yml + - id: spell-check + name: Spelling checks + kind: ci + required: true + source: .github/workflows/spell-check.yml + - id: provenance-block + name: Provenance block present and strict check passes + kind: manual + required: true + source: "spec:15. The provenance block#L793-L877" + bookkeeping: + - on: accepted + action: notify-contacts + params: {note: "mention the validated project contacts on the intake issue before drafting (HC-6)", source: "spec:12. Lifecycle bindings#L482-L636"} + - on: ready + action: mention-stakeholders + params: {note: "mention the stakeholder set frozen at acceptance with the participant guide link", source: "spec:12. Lifecycle bindings#L482-L636"} + - on: merged + action: open-next-phase-tracking-issue + params: {note: "open the next phase's tracking issue labeled phase/* and needs-triage (phase C when skip-implementation is set); close the intake as completed after the final phase", source: "spec:12. Lifecycle bindings#L482-L636"} + labels: + entries: + - {name: Docs analysis, color: c2e0c6, description: CNCF technical documentation assistance} + - {name: needs-triage, color: EDEDED, description: Assessment request awaiting triage, state: requested} + - {name: triage/accepted, color: C2E0C6, description: Assessment request accepted, state: accepted} + - {name: triage/declined, color: F9D0C4, description: "Assessment request declined, reason posted", state: declined} + - {name: phase/assessment, color: C5DEF5, description: "Assessment phase A: analysis"} + - {name: phase/implementation, color: 84B6EB, description: "Assessment phase B: implementation plan"} + - {name: phase/backlog, color: 1D76DB, description: "Assessment phase C: issue backlog"} + - {name: verified, color: 0E8A16, description: Verification complete for this deliverable PR, state: in-review} + - {name: confirmed, color: 006B75, description: Stakeholder confirmation recorded for this deliverable PR, state: awaiting-merge} + - {name: skip-implementation, color: D4C5F9, description: Phase B skipped; phase A merge opens phase C} + transitions: + # issues + - {from: requested, to: accepted, on: accepted, gateRef: accept, note: "spec §12 Accept"} + - {from: requested, to: declined, on: declined, gateRef: decline, note: "spec §12 Accept: /decline"} + - {from: requested, to: closed, on: closed, note: "withdrawn by the project"} + - {from: accepted, to: draft-open, on: draft-opened, note: "spec §12 Draft"} + - {from: accepted, to: escalated, on: held, note: "spec §5 failure path: escalated to the administrator"} + - {from: accepted, to: closed, on: closed, note: "tracking issue declined or assessment ended"} + - {from: draft-open, to: accepted, on: draft-withdrawn, note: "draft discarded: restart, hand-write or stop"} + - {from: draft-open, to: escalated, on: held, note: "spec §5 failure path"} + - {from: draft-open, to: closed, on: closed, note: "phase merged (next phase opens) or aborted"} + - {from: escalated, to: accepted, on: released, note: "administrator released the hold"} + - {from: escalated, to: closed, on: closed, gateRef: abort, note: "spec §12 Failure and abort"} + # pull requests + - {from: drafting, to: in-review, on: verified, gateRef: verified, note: "spec §12 Review: verified label is the gate"} + - {from: drafting, to: discarded, on: closed, gateRef: discard} + - {from: in-review, to: stakeholder-review, on: ready, gateRef: ready, note: "spec §12 Stakeholder review: /ready"} + - {from: in-review, to: discarded, on: closed, gateRef: discard} + - {from: checks-failing, to: stakeholder-review, on: checks-green} + - {from: checks-failing, to: discarded, on: closed, gateRef: discard} + - {from: stakeholder-review, to: awaiting-merge, on: confirmed, gateRef: confirm, note: "spec §12: /confirm from the frozen stakeholder set"} + - {from: stakeholder-review, to: checks-failing, on: checks-red} + - {from: stakeholder-review, to: discarded, on: closed, gateRef: discard} + - {from: awaiting-merge, to: merged, on: merged, gateRef: approve, note: "spec §5 step 6 / §12 Merge"} + - {from: awaiting-merge, to: checks-failing, on: checks-red} + - {from: awaiting-merge, to: discarded, on: closed, gateRef: discard} + derivation: + mode: rules + rules: + # issues (intake umbrella and phase tracking issues) + - id: declined + kind: issue + state: declined + when: + - {fact: open, op: "false"} + - {fact: labels, op: has, value: triage/declined} + reason: declined by a technical writer (triage/declined) + - id: closed + kind: issue + state: closed + when: + - {fact: open, op: "false"} + reason: assessment ended (completed at the final merge, or aborted) + - id: escalated + kind: issue + state: escalated + when: + - {fact: hold, op: "true"} + reason: on hold; escalated to the administrator/platform owner + - id: draft-open + kind: issue + state: draft-open + when: + - {fact: labels, op: has, value: triage/accepted} + - {fact: issue.covered, op: "true"} + reason: a draft PR covers this phase + - id: accepted + kind: issue + state: accepted + when: + - {fact: labels, op: has, value: triage/accepted} + reason: accepted; awaiting a draft + - id: requested + kind: issue + state: requested + when: + - {fact: labels, op: has, value: needs-triage} + reason: filed; awaiting triage + # pull requests (the deliverable carries the Docs analysis label) + - id: merged + kind: pr + state: merged + when: + - {fact: merged, op: "true"} + - {fact: labels, op: has, value: Docs analysis} + reason: deliverable merged by the approver + - id: discarded + kind: pr + state: discarded + when: + - {fact: open, op: "false"} + - {fact: labels, op: has, value: Docs analysis} + reason: draft discarded (closed unmerged) + - id: in-review-draft + kind: pr + state: in-review + when: + - {fact: draft, op: "true"} + - {fact: labels, op: has, value: verified} + - {fact: labels, op: has, value: Docs analysis} + reason: verified; the reviewer refines and checks findings against source + - id: drafting + kind: pr + state: drafting + when: + - {fact: draft, op: "true"} + - {fact: labels, op: has, value: Docs analysis} + reason: AI-drafted deliverable in progress; verification not yet recorded + - id: checks-failing + kind: pr + state: checks-failing + when: + - {fact: pr.checks, op: is, value: red} + - {fact: labels, op: has, value: Docs analysis} + reason: a format, link or spelling check is red + - id: awaiting-merge + kind: pr + state: awaiting-merge + when: + - {fact: labels, op: has, value: confirmed} + - {fact: labels, op: has, value: Docs analysis} + reason: stakeholder confirmation recorded; awaiting the approver + - id: stakeholder-review + kind: pr + state: stakeholder-review + when: + - {fact: labels, op: has, value: verified} + - {fact: labels, op: has, value: Docs analysis} + reason: ready for review; awaiting /confirm from a stakeholder + - id: in-review-unverified + kind: pr + state: in-review + when: + - {fact: labels, op: has, value: Docs analysis} + reason: ready without the verified label; verification is still owed + pipelines: + - id: gha-github-workflows-format-check + name: Format checks + system: github-actions + path: .github/workflows/format-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: format-check, name: FILE FORMAT, kind: job, runsOn: [ubuntu-latest], level: 0} + - {id: markdown-linter, name: MARKDOWN linter, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + - id: gha-github-workflows-link-check + name: Link checks + system: github-actions + path: .github/workflows/link-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: link-check, name: LINK checking, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + - id: gha-github-workflows-spell-check + name: Spelling checks + system: github-actions + path: .github/workflows/spell-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: spelling-check, name: SPELLING check, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + coverage: + files: 3 + parsed: 3 diff --git a/src/pkg/hub/marketplace/curated/curated.go b/src/pkg/hub/marketplace/curated/curated.go new file mode 100644 index 0000000000..7a415b43de --- /dev/null +++ b/src/pkg/hub/marketplace/curated/curated.go @@ -0,0 +1,96 @@ +// Package curated embeds the hub marketplace's curated workflow models +// (hivecommons/hive#17620 phase 7). Each *.yaml here is one reviewed model +// at one version; the hub seeds the catalogue from them at startup and on +// every refresh run, publishing each name@version that is not yet present. +// Entries are immutable: a change is a new file with a new spec.meta.version, +// never an edit in place. +package curated + +import ( + "bytes" + "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +//go:embed *.yaml +var files embed.FS + +// Errors a curated file can fail with. +var ( + ErrNotCurated = errors.New("curated: spec.meta.provenance must be curated") + ErrNoVersion = errors.New("curated: spec.meta.version is empty") + ErrInferred = errors.New("curated: model still carries inferred elements") +) + +// Entry is one embedded curated model. +type Entry struct { + Path string + Model *workflowmodel.Model +} + +// Decode parses one curated model document (YAML or JSON) strictly and +// checks the curated invariants: provenance curated, a version, and no +// element left at confidence inferred. It does not compile the model; the +// seed does that through bundle.Export. +func Decode(data []byte) (*workflowmodel.Model, error) { + var doc interface{} + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("curated: yaml: %w", err) + } + raw, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("curated: yaml to json: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + var m workflowmodel.Model + if err := dec.Decode(&m); err != nil { + return nil, fmt.Errorf("curated: not a WorkflowModel: %w", err) + } + if m.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + return nil, fmt.Errorf("%w: got %q", ErrNotCurated, m.Spec.Meta.Provenance) + } + if strings.TrimSpace(m.Spec.Meta.Version) == "" { + return nil, ErrNoVersion + } + if n := workflowmodel.Inferred(&m); n > 0 { + return nil, fmt.Errorf("%w: %d unconfirmed", ErrInferred, n) + } + return &m, nil +} + +// Load decodes every embedded *.yaml in path order. Files that fail are +// reported in errs (one per file, prefixed with the path) and skipped, so +// one bad file never hides the others. +func Load() (entries []Entry, errs []error) { + return load(files) +} + +func load(fsys fs.FS) (entries []Entry, errs []error) { + // The pattern is a constant, so Glob cannot fail on it. + names, _ := fs.Glob(fsys, "*.yaml") + sort.Strings(names) + for _, name := range names { + data, err := fs.ReadFile(fsys, name) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", name, err)) + continue + } + m, err := Decode(data) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", name, err)) + continue + } + entries = append(entries, Entry{Path: name, Model: m}) + } + return entries, errs +} diff --git a/src/pkg/hub/marketplace/curated/curated_test.go b/src/pkg/hub/marketplace/curated/curated_test.go new file mode 100644 index 0000000000..81700c3232 --- /dev/null +++ b/src/pkg/hub/marketplace/curated/curated_test.go @@ -0,0 +1,212 @@ +package curated_test + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/authorkind" + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/stageengine" + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +// TestLoadEmbedded is the gate every curated file must pass: strict +// decode, provenance curated, a version, zero compile diagnostics of any +// severity, nothing left inferred, and a clean bundle export at tier +// curated. +func TestLoadEmbedded(t *testing.T) { + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatalf("load errors: %v", errs) + } + if len(entries) == 0 { + t.Fatal("no curated models embedded") + } + seen := map[string]bool{} + for _, e := range entries { + m := e.Model + key := m.Metadata.Name + "@" + m.Spec.Meta.Version + if seen[key] { + t.Fatalf("%s: duplicate %s", e.Path, key) + } + seen[key] = true + if m.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + t.Fatalf("%s: provenance %q", e.Path, m.Spec.Meta.Provenance) + } + if n := workflowmodel.Inferred(m); n != 0 { + t.Fatalf("%s: %d inferred elements unconfirmed", e.Path, n) + } + _, diags, err := workflowmodel.Compile(m) + if err != nil || len(diags) != 0 { + t.Fatalf("%s: compile err=%v diags=%+v", e.Path, err, diags) + } + b, err := bundle.Export(m, "test", testNow) + if err != nil { + t.Fatalf("%s: export: %v", e.Path, err) + } + if b.Tier != bundle.TierCurated { + t.Fatalf("%s: tier %q", e.Path, b.Tier) + } + } + if !seen["cncf-techdocs-assessment@curated-2026.11.1"] { + t.Fatalf("techdocs model missing: %v", seen) + } +} + +func TestDecodeRejects(t *testing.T) { + cases := []struct { + name string + doc string + want error + }{ + {"bad yaml", "a: [", nil}, + {"unknown field", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nbogus: 1\n", nil}, + {"not curated", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {version: v1, provenance: ingested}}\n", curated.ErrNotCurated}, + {"no version", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {provenance: curated}}\n", curated.ErrNoVersion}, + {"inferred", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {version: v1, provenance: curated}, states: [{id: x, lifecycle: {confidence: inferred}}]}\n", curated.ErrInferred}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := curated.Decode([]byte(tc.doc)) + if err == nil { + t.Fatal("expected error") + } + if tc.want != nil && !errors.Is(err, tc.want) { + t.Fatalf("err %v, want %v", err, tc.want) + } + }) + } +} + +func techdocs(t *testing.T) *workflowmodel.Profile { + t.Helper() + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + for _, e := range entries { + if e.Model.Metadata.Name == "cncf-techdocs-assessment" { + p, _, err := workflowmodel.Compile(e.Model) + if err != nil { + t.Fatal(err) + } + return p + } + } + t.Fatal("cncf-techdocs-assessment not embedded") + return nil +} + +func issue(labels ...string) stageengine.Facts { + return stageengine.Facts{ + Ref: stageengine.Ref{Owner: "cncf", Repo: "techdocs", Number: 1, Kind: stagelabels.IssueItem}, + Open: true, + Labels: labels, + Author: stageengine.AuthorFacts{Kind: authorkind.Human, Login: "maintainer"}, + Issue: &stageengine.IssueFacts{}, + } +} + +func pr(labels ...string) stageengine.Facts { + return stageengine.Facts{ + Ref: stageengine.Ref{Owner: "cncf", Repo: "techdocs", Number: 2, Kind: stagelabels.PRItem}, + Open: true, + Labels: append([]string{"Docs analysis"}, labels...), + Author: stageengine.AuthorFacts{Kind: authorkind.Hive, Login: "hive"}, + PR: &stageengine.PRFacts{Checks: stagelabels.CheckSummary{Known: true, Total: 3, Green: 3}, DCO: stagelabels.DCOYes}, + } +} + +// TestTechDocsRules walks an assessment through the rules evaluator the +// way Nate's spec describes it: request → accepted → draft → verified → +// stakeholder review → confirmed → merged, plus the declined and held +// branches. +func TestTechDocsRules(t *testing.T) { + p := techdocs(t) + issues := stageengine.DeriveFromModel(p, stagelabels.IssueItem) + prs := stageengine.DeriveFromModel(p, stagelabels.PRItem) + + cases := []struct { + name string + d stageengine.Deriver + f func() stageengine.Facts + stage stagelabels.Stage + term bool + }{ + {"new request", issues, func() stageengine.Facts { return issue("needs-triage") }, stagelabels.Actionable, false}, + {"accepted", issues, func() stageengine.Facts { return issue("triage/accepted") }, stagelabels.Claimed, false}, + {"draft posted covers the request", issues, func() stageengine.Facts { + f := issue("triage/accepted", "phase/analysis") + f.Issue.CoveringPR = &stageengine.Ref{Owner: "cncf", Repo: "techdocs", Number: 2, Kind: stagelabels.PRItem} + return f + }, stagelabels.CoveredByPR, false}, + {"escalated holds", issues, func() stageengine.Facts { + f := issue("triage/accepted") + f.Park.Holds = []string{"hold"} + return f + }, stagelabels.IssueNeedsDecision, false}, + {"declined", issues, func() stageengine.Facts { f := issue("triage/declined"); f.Open = false; return f }, stagelabels.None, true}, + {"closed", issues, func() stageengine.Facts { f := issue("triage/accepted"); f.Open = false; return f }, stagelabels.None, true}, + {"drafting", prs, func() stageengine.Facts { f := pr(); f.Draft = true; return f }, stagelabels.Fixing, false}, + {"verified draft in review", prs, func() stageengine.Facts { f := pr("verified"); f.Draft = true; return f }, stagelabels.ReviewPending, false}, + {"ready but unverified stays in review", prs, func() stageengine.Facts { return pr() }, stagelabels.ReviewPending, false}, + {"checks failing", prs, func() stageengine.Facts { + f := pr("verified") + f.PR.Checks.Red = []string{"spell-check"} + return f + }, stagelabels.NeedsAuthor, false}, + {"stakeholder review", prs, func() stageengine.Facts { return pr("verified") }, stagelabels.Reviewed, false}, + {"approved awaits merge", prs, func() stageengine.Facts { return pr("verified", "confirmed") }, stagelabels.ReadyForHumanMerge, false}, + {"merged", prs, func() stageengine.Facts { f := pr(); f.Open, f.Merged = false, true; return f }, stagelabels.None, true}, + {"discarded", prs, func() stageengine.Facts { f := pr(); f.Open = false; return f }, stagelabels.None, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := tc.d.Derive(tc.f()) + if got.Abstain { + t.Fatalf("abstained: %s", got.Reason) + } + if got.Terminal != tc.term || got.Stage != tc.stage { + t.Fatalf("got %+v; want stage %s terminal %v", got, tc.stage, tc.term) + } + }) + } + t.Run("unrelated issue abstains", func(t *testing.T) { + if got := issues.Derive(issue("bug")); !got.Abstain { + t.Fatalf("%+v", got) + } + }) + t.Run("unlabelled PR abstains", func(t *testing.T) { + f := pr() + f.Labels = nil + if got := prs.Derive(f); !got.Abstain { + t.Fatalf("%+v", got) + } + }) + t.Run("replies resolve to gates", func(t *testing.T) { + for state, verb := range map[string]string{"requested": "/accept", "stakeholder-review": "/confirm", "awaiting-merge": "/approve"} { + r, ok := p.Reply(state, verb) + if !ok || r.To == "" { + t.Fatalf("%s %s: ok=%v %+v", state, verb, ok, r) + } + } + if _, ok := p.Reply("requested", "/approve"); ok { + t.Fatal("approve offered out of requested") + } + }) +} + +var testNow = time.Date(2026, 3, 14, 0, 0, 0, 0, time.UTC) + +func TestEntriesSorted(t *testing.T) { + entries, _ := curated.Load() + for i := 1; i < len(entries); i++ { + if strings.Compare(entries[i-1].Path, entries[i].Path) > 0 { + t.Fatalf("not sorted: %s > %s", entries[i-1].Path, entries[i].Path) + } + } +} diff --git a/src/pkg/hub/marketplace/curated/load_test.go b/src/pkg/hub/marketplace/curated/load_test.go new file mode 100644 index 0000000000..d59018c33f --- /dev/null +++ b/src/pkg/hub/marketplace/curated/load_test.go @@ -0,0 +1,46 @@ +package curated + +import ( + "errors" + "io/fs" + "testing" + "testing/fstest" +) + +func TestLoadSkipsBadFiles(t *testing.T) { + good, err := files.ReadFile("cncf-techdocs-assessment.yaml") + if err != nil { + t.Fatal(err) + } + fsys := fstest.MapFS{ + "a-bad.yaml": {Data: []byte("a: [")}, + "b-good.yaml": {Data: good}, + "c-wrong.yaml": {Data: []byte("apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {version: v1, provenance: ingested}}\n")}, + "ignored.json": {Data: []byte("{")}, + } + entries, errs := load(fsys) + if len(entries) != 1 || entries[0].Path != "b-good.yaml" { + t.Fatalf("entries = %+v", entries) + } + if len(errs) != 2 { + t.Fatalf("errs = %v", errs) + } +} + +// unreadableFS globs one file whose read fails. +type unreadableFS struct{ fstest.MapFS } + +func (u unreadableFS) Open(name string) (fs.File, error) { + if name == "x.yaml" { + return nil, errors.New("boom") + } + return u.MapFS.Open(name) +} + +func TestLoadReadError(t *testing.T) { + fsys := unreadableFS{fstest.MapFS{"x.yaml": {Data: []byte("a: 1")}}} + entries, errs := load(fsys) + if len(entries) != 0 || len(errs) != 1 { + t.Fatalf("entries=%v errs=%v", entries, errs) + } +} diff --git a/src/pkg/hub/marketplace_curated.go b/src/pkg/hub/marketplace_curated.go new file mode 100644 index 0000000000..6ca8deb850 --- /dev/null +++ b/src/pkg/hub/marketplace_curated.go @@ -0,0 +1,56 @@ +package hub + +import ( + "time" + + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +// RFC #17620 phase 7: curated seeding. +// +// The hub ships reviewed models under pkg/hub/marketplace/curated/*.yaml. +// seedCuratedModels compiles each, wraps it in a bundle at tier curated and +// publishes it when that name@version is not yet in the catalogue. Entries +// are immutable, so a changed file with an unchanged version is reported as +// a conflict rather than overwriting; a new version is a new file. Ingest +// never touches curated versions; marketplaceNormaliseIndex flags them +// newerIngestAvailable when a later ingested-unreviewed version exists. + +// marketplaceCuratedBy is the publisher recorded on seeded bundles. +const marketplaceCuratedBy = "curated-seed" + +// marketplaceCuratedLoad is swapped in tests to inject bad or extra models. +var marketplaceCuratedLoad = curated.Load + +// seedCuratedModels publishes every embedded curated model that is absent +// from the catalogue. It returns the keys it published and the errors it +// met; nothing here fails the caller — a bad curated file is logged and +// skipped so the rest of the catalogue still seeds. +func (s *HubServer) seedCuratedModels(st *marketplaceStore, now time.Time) (published []string, errs []error) { + entries, loadErrs := marketplaceCuratedLoad() + for _, err := range loadErrs { + s.logger.Error("marketplace: curated model refused", "error", err) + errs = append(errs, err) + } + for _, e := range entries { + b, err := bundle.Export(e.Model, marketplaceCuratedBy, now) + if err != nil { + s.logger.Error("marketplace: curated model does not compile", "path", e.Path, "error", err) + errs = append(errs, err) + continue + } + b.Tier = bundle.TierCurated + entry, created, err := st.Publish(b, marketplaceCuratedBy, now, "") + if err != nil { + s.logger.Error("marketplace: curated model not published", "path", e.Path, "key", b.Key(), "error", err) + errs = append(errs, err) + continue + } + if created { + s.logger.Info("marketplace: curated model published", "path", e.Path, "key", entry.Key()) + published = append(published, entry.Key()) + } + } + return published, errs +} diff --git a/src/pkg/hub/marketplace_curated_test.go b/src/pkg/hub/marketplace_curated_test.go new file mode 100644 index 0000000000..b08a0786cb --- /dev/null +++ b/src/pkg/hub/marketplace_curated_test.go @@ -0,0 +1,159 @@ +package hub + +import ( + "bytes" + "context" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +const mktTechDocsKey = "cncf-techdocs-assessment@curated-2026.11.1" + +// TestMarketplaceCuratedSeedPublishesAndIsIdempotent: the embedded curated +// models land at tier curated on the first seed; a second seed publishes +// nothing and leaves exactly one version per model. +func TestMarketplaceCuratedSeedPublishesAndIsIdempotent(t *testing.T) { + s := marketplaceTestHub(t, nil) + st := s.marketplace() + now := time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC) + + published, errs := s.seedCuratedModels(st, now) + if len(errs) != 0 { + t.Fatalf("seed errors: %v", errs) + } + if len(published) != mktCuratedCount(t) || published[0] != mktTechDocsKey { + t.Fatalf("published = %v", published) + } + raw, err := st.Get("cncf-techdocs-assessment", "curated-2026.11.1") + if err != nil { + t.Fatal(err) + } + b, err := bundle.Decode(raw) + if err != nil { + t.Fatal(err) + } + if b.Tier != bundle.TierCurated || b.CreatedBy != marketplaceCuratedBy || b.Model.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + t.Fatalf("bundle: tier=%s by=%s prov=%s", b.Tier, b.CreatedBy, b.Model.Spec.Meta.Provenance) + } + if b.Model.Spec.Scope.Repository != "cncf/techdocs" { + t.Fatalf("scope: %+v", b.Model.Spec.Scope) + } + + published, errs = s.seedCuratedModels(st, now.Add(time.Hour)) + if len(published) != 0 || len(errs) != 0 { + t.Fatalf("second seed: published=%v errs=%v", published, errs) + } + vs, err := st.Versions("cncf-techdocs-assessment") + if err != nil || len(vs) != 1 { + t.Fatalf("versions: %v %d", err, len(vs)) + } + idx, _, _, err := st.Index() + if err != nil { + t.Fatal(err) + } + for _, e := range idx.Entries { + if e.Name == "cncf-techdocs-assessment" && (e.Tier != bundle.TierCurated || e.NewerIngestAvailable) { + t.Fatalf("index entry: %+v", e) + } + } +} + +// TestMarketplaceCuratedSeedRefusesBadModels: a loader error and a model +// that fails to compile are logged and skipped; the good model still seeds +// and nothing panics. +func TestMarketplaceCuratedSeedRefusesBadModels(t *testing.T) { + s := marketplaceTestHub(t, nil) + var buf bytes.Buffer + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + good, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + broken := *good[0].Model + broken.Metadata.Name = "broken-curated" + broken.Spec.Columns = nil // compile refuses: states without columns + old := marketplaceCuratedLoad + t.Cleanup(func() { marketplaceCuratedLoad = old }) + marketplaceCuratedLoad = func() ([]curated.Entry, []error) { + return append([]curated.Entry{{Path: "broken.yaml", Model: &broken}}, good...), + []error{errors.New("bad.yaml: curated: yaml: oops")} + } + + published, seedErrs := s.seedCuratedModels(s.marketplace(), time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC)) + if len(published) != len(good) || len(seedErrs) != 2 { + t.Fatalf("published=%v errs=%v", published, seedErrs) + } + logs := buf.String() + for _, want := range []string{"curated model refused", "bad.yaml", "does not compile", "broken.yaml", "curated model published"} { + if !strings.Contains(logs, want) { + t.Errorf("log missing %q:\n%s", want, logs) + } + } + if vs, err := s.marketplace().Versions("broken-curated"); err == nil && len(vs) != 0 { + t.Fatalf("broken model must not be published: %v", vs) + } + + // A refresh run surfaces the same errors without aborting ingest. + res, err := s.runMarketplaceRefresh(context.Background(), []string{}, "test") + if err != nil { + t.Fatal(err) + } + if len(res.Curated) != 0 || len(res.Errors) != 2 || !strings.HasPrefix(res.Errors[0], "curated: ") { + t.Fatalf("refresh: %+v", res) + } +} + +// TestMarketplaceCuratedSeedConflictIsImmutable: editing a curated file +// without bumping its version is refused as a conflict; the stored bundle +// stays as first published. +func TestMarketplaceCuratedSeedConflictIsImmutable(t *testing.T) { + s := marketplaceTestHub(t, nil) + st := s.marketplace() + now := time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC) + if _, errs := s.seedCuratedModels(st, now); len(errs) != 0 { + t.Fatal(errs) + } + good, _ := curated.Load() + edited := *good[0].Model + edited.Spec.Meta.Description = "edited in place without a version bump" + old := marketplaceCuratedLoad + t.Cleanup(func() { marketplaceCuratedLoad = old }) + marketplaceCuratedLoad = func() ([]curated.Entry, []error) { + return []curated.Entry{{Path: good[0].Path, Model: &edited}}, nil + } + published, errs := s.seedCuratedModels(st, now) + if len(published) != 0 || len(errs) != 1 || !errors.Is(errs[0], errMarketplaceConflict) { + t.Fatalf("published=%v errs=%v", published, errs) + } +} + +// TestMarketplaceStartRefreshSeedsCurated: startup seeds curated models +// before any ingest, with no GitHub token configured. +func TestMarketplaceStartRefreshSeedsCurated(t *testing.T) { + s := marketplaceTestHub(t, nil) + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("HIVE_HUB_GITHUB_TOKEN", "") + t.Setenv(hubMarketplaceScheduleEnv, "") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { s.StartMarketplaceRefresh(ctx); close(done) }() + testutil.Eventually(t, 5*time.Second, func() bool { + vs, err := s.marketplace().Versions("cncf-techdocs-assessment") + return err == nil && len(vs) == 1 + }, "curated model not seeded at startup") + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("StartMarketplaceRefresh did not stop on cancel") + } +} diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index c6eb57af02..baf5d208a7 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -22,8 +22,9 @@ import ( // RFC #17620 phase 7: the marketplace's ingest refresh. // -// Every run walks the CNCF survey corpus (pkg/workflowmodel/ingest/corpus) -// plus any operator-added repos, fetches each default branch, hashes the +// Every run first seeds the curated models (marketplace_curated.go), then +// walks the CNCF survey corpus (pkg/workflowmodel/ingest/corpus, including +// its operator-added extra.json) plus HIVE_HUB_MARKETPLACE_EXTRA_REPOS, fetches each default branch, hashes the // pipeline-file set and compares it with the last recorded run for that // repository. Unchanged → no-op. Changed → Lifecycle → Compile → bundle → // publish as `ingested-YYYYMMDD.N`, tier ingested-unreviewed. Curated @@ -82,9 +83,12 @@ type marketplaceRefreshResult struct { Repos int `json:"repos"` Unchanged int `json:"unchanged"` Published []string `json:"published"` - Skipped int `json:"skipped"` - Errors []string `json:"errors"` - Trigger string `json:"trigger"` + // Curated lists the curated name@version keys this run seeded (empty + // once they are all present; curated entries are immutable). + Curated []string `json:"curated,omitempty"` + Skipped int `json:"skipped"` + Errors []string `json:"errors"` + Trigger string `json:"trigger"` } // marketplaceRefresher owns the run lock and the last-run summary. @@ -219,6 +223,11 @@ func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, t } res := &marketplaceRefreshResult{StartedAt: rf.now().UTC(), Trigger: trigger, Published: []string{}, Errors: []string{}} s.logger.Info("marketplace refresh started", "trigger", trigger, "repos", len(repos)) + seeded, seedErrs := s.seedCuratedModels(st, rf.now()) + res.Curated = seeded + for _, err := range seedErrs { + res.Errors = append(res.Errors, "curated: "+err.Error()) + } for i, repo := range repos { if ctx.Err() != nil { res.Errors = append(res.Errors, "cancelled: "+ctx.Err().Error()) @@ -492,6 +501,9 @@ func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { rf.mu.Unlock() schedule := marketplaceSchedule() s.logger.Info("marketplace refresh started", "schedule_utc", schedule, "corpus_repos", len(marketplaceRepos())) + // Curated models need no GitHub token and no delay: seed them first so + // the catalogue is never empty while the ingest crawl is pending. + s.seedCuratedModels(s.marketplace(), rf.now()) if due := s.marketplaceDueRepos(); len(due) > 0 { s.logger.Warn("marketplace catalogue has repositories due for ingest: resuming in the background", "due", len(due), "delay", marketplaceSeedDelay.String()) marketplaceSleep(ctx, marketplaceSeedDelay) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index c3672cd46e..c3b8e424de 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/bundle" ) @@ -48,6 +49,16 @@ func marketplaceTestHub(t *testing.T, fetch marketplaceTreeFetcher) *HubServer { return s } +// mktCuratedCount is how many curated models the hub embeds and seeds. +func mktCuratedCount(t *testing.T) int { + t.Helper() + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + return len(entries) +} + func mktModel(t *testing.T) *workflowmodel.Model { t.Helper() raw, err := os.ReadFile("../workflowmodel/testdata/models/techdocs.json") @@ -431,11 +442,17 @@ func TestMarketplaceRefreshNoopVsNewVersion(t *testing.T) { t.Errorf("version naming: %s", k) } } + if len(res.Curated) != mktCuratedCount(t) { + t.Fatalf("curated seeded on first run: %v", res.Curated) + } idx, _, _, err := s.marketplace().Index() - if err != nil || len(idx.Entries) != 2 { + if err != nil || len(idx.Entries) != 2+mktCuratedCount(t) { t.Fatalf("index after first run: %v %d", err, len(idx.Entries)) } for _, e := range idx.Entries { + if e.Tier == bundle.TierCurated { + continue + } if e.Tier != bundle.TierIngestedUnreviewed || e.SourceSHA == "" { t.Errorf("entry: %+v", e) } @@ -446,7 +463,7 @@ func TestMarketplaceRefreshNoopVsNewVersion(t *testing.T) { if err != nil { t.Fatal(err) } - if len(res.Published) != 0 || res.Unchanged != 3 || len(res.Errors) != 1 { + if len(res.Published) != 0 || len(res.Curated) != 0 || res.Unchanged != 3 || len(res.Errors) != 1 { t.Fatalf("second run: %+v", res) } @@ -588,9 +605,12 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { t.Fatal(err) } - if status["entries"].(float64) != 1 || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { + if int(status["entries"].(float64)) != 1+mktCuratedCount(t) || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { t.Fatalf("status body: %s", rec.Body.String()) } + if byTier, _ := status["byTier"].(map[string]interface{}); int(byTier[bundle.TierCurated].(float64)) != mktCuratedCount(t) { + t.Fatalf("curated tier not counted: %s", rec.Body.String()) + } if status["pendingRepos"].(float64) != 0 || status["lastRefreshErrors"].(float64) != 0 { t.Fatalf("pending/lastErrors: %s", rec.Body.String()) } @@ -641,7 +661,8 @@ func TestMarketplaceSeedResumesAfterRestart(t *testing.T) { if _, err := s.runMarketplaceRefresh(context.Background(), []string{"konflux-ci/konflux-ci"}, "seed"); err != nil { t.Fatal(err) } - if idx, _, _, _ := s.marketplace().Index(); len(idx.Entries) != 1 { + // One ingested entry plus the curated models every run seeds. + if idx, _, _, _ := s.marketplace().Index(); len(idx.Entries) != 1+mktCuratedCount(t) { t.Fatalf("precondition: %d entries", len(idx.Entries)) } due := s.marketplaceDueRepos() diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus.go b/src/pkg/workflowmodel/ingest/corpus/corpus.go index a9139905f8..0c87d12c58 100644 --- a/src/pkg/workflowmodel/ingest/corpus/corpus.go +++ b/src/pkg/workflowmodel/ingest/corpus/corpus.go @@ -4,11 +4,19 @@ // refresh seeds its `ingested-unreviewed` tier from this list, and // cmd/workflow-survey falls back to it when a corpus directory carries no // projects.json of its own. +// +// corpus.json is the survey's projects.json and is regenerated from the +// CNCF landscape; do not hand-edit it. extra.json is the operator-added +// list of repositories that are not landscape projects but whose workflow +// the marketplace should still ingest (cncf/techdocs, for one). Add a repo +// there: {"name", "category", "repos": ["owner/repo"], "why"}. Extra +// entries carry no maturity. package corpus import ( _ "embed" "encoding/json" + "fmt" "sort" "strings" "sync" @@ -17,6 +25,9 @@ import ( //go:embed corpus.json var corpusJSON []byte +//go:embed extra.json +var extraJSON []byte + // Project is one surveyed CNCF project. type Project struct { Name string `json:"name"` @@ -25,29 +36,44 @@ type Project struct { Repos []string `json:"repos"` } +// Extra is one operator-added, non-landscape repository group (extra.json). +type Extra struct { + Name string `json:"name"` + Category string `json:"category,omitempty"` + Repos []string `json:"repos"` + // Why records the reason the entry is in the corpus. + Why string `json:"why,omitempty"` +} + var ( - loadOnce sync.Once - loaded []Project - loadErr error + loadOnce sync.Once + loaded []Project + loadedExtra []Extra + loadErr error ) -func load() ([]Project, error) { +func load() ([]Project, []Extra, error) { loadOnce.Do(func() { var ps []Project if err := json.Unmarshal(corpusJSON, &ps); err != nil { - loadErr = err + loadErr = fmt.Errorf("corpus.json: %w", err) return } - loaded = ps + var xs []Extra + if err := json.Unmarshal(extraJSON, &xs); err != nil { + loadErr = fmt.Errorf("extra.json: %w", err) + return + } + loaded, loadedExtra = ps, xs }) - return loaded, loadErr + return loaded, loadedExtra, loadErr } // Projects returns a copy of the embedded project list, sorted by name. func Projects() []Project { - ps, err := load() + ps, _, err := load() if err != nil { - panic("corpus: embedded corpus.json is invalid: " + err.Error()) + panic("corpus: embedded corpus is invalid: " + err.Error()) } out := make([]Project, len(ps)) for i, p := range ps { @@ -57,19 +83,42 @@ func Projects() []Project { return out } -// Repos returns every distinct `owner/repo` in the corpus, lower-cased and -// sorted, so callers iterate deterministically. +// Extras returns a copy of the operator-added non-landscape entries. +func Extras() []Extra { + _, xs, err := load() + if err != nil { + panic("corpus: embedded corpus is invalid: " + err.Error()) + } + out := make([]Extra, len(xs)) + for i, x := range xs { + out[i] = x + out[i].Repos = append([]string(nil), x.Repos...) + } + return out +} + +// Repos returns every distinct `owner/repo` in the corpus — survey +// projects plus extras — lower-cased and sorted, so callers iterate +// deterministically. func Repos() []string { seen := map[string]bool{} var out []string + add := func(r string) { + r = strings.ToLower(strings.TrimSpace(r)) + if r == "" || seen[r] { + return + } + seen[r] = true + out = append(out, r) + } for _, p := range Projects() { for _, r := range p.Repos { - r = strings.ToLower(strings.TrimSpace(r)) - if r == "" || seen[r] { - continue - } - seen[r] = true - out = append(out, r) + add(r) + } + } + for _, x := range Extras() { + for _, r := range x.Repos { + add(r) } } sort.Strings(out) diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus_test.go b/src/pkg/workflowmodel/ingest/corpus/corpus_test.go index 2892626047..787778cbb5 100644 --- a/src/pkg/workflowmodel/ingest/corpus/corpus_test.go +++ b/src/pkg/workflowmodel/ingest/corpus/corpus_test.go @@ -38,6 +38,20 @@ func TestProjectsAndRepos(t *testing.T) { if m["project-akri/akri"] != "sandbox" { t.Fatalf("maturity = %q", m["project-akri/akri"]) } + if !seen["cncf/techdocs"] { + t.Fatal("expected the extra.json repo cncf/techdocs in Repos()") + } + xs := Extras() + if len(xs) == 0 || xs[0].Repos[0] != "cncf/techdocs" || xs[0].Why == "" { + t.Fatalf("extras = %+v", xs) + } + xs[0].Repos[0] = "tampered" + if Extras()[0].Repos[0] == "tampered" { + t.Fatal("Extras leaked internal slice") + } + if _, ok := m["cncf/techdocs"]; ok { + t.Fatal("extras must not carry a maturity") + } // Projects() hands out copies. ps[0].Repos[0] = "tampered" if Projects()[0].Repos[0] == "tampered" { @@ -46,15 +60,27 @@ func TestProjectsAndRepos(t *testing.T) { } func TestLoadInvalid(t *testing.T) { - old := corpusJSON - t.Cleanup(func() { corpusJSON = old; loadOnce = sync.Once{}; loaded, loadErr = nil, nil }) - loadOnce = sync.Once{} - loaded, loadErr = nil, nil - corpusJSON = []byte("{not json") - defer func() { - if recover() == nil { - t.Fatal("expected panic on invalid embedded corpus") - } - }() - Projects() + reset := func() { loadOnce = sync.Once{}; loaded, loadedExtra, loadErr = nil, nil, nil } + oldCorpus, oldExtra := corpusJSON, extraJSON + t.Cleanup(func() { corpusJSON, extraJSON = oldCorpus, oldExtra; reset() }) + expectPanic := func(t *testing.T, f func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Fatal("expected panic on invalid embedded corpus") + } + }() + f() + } + t.Run("corpus.json", func(t *testing.T) { + reset() + corpusJSON = []byte("{not json") + expectPanic(t, func() { Projects() }) + }) + t.Run("extra.json", func(t *testing.T) { + reset() + corpusJSON = oldCorpus + extraJSON = []byte("{not json") + expectPanic(t, func() { Extras() }) + }) } diff --git a/src/pkg/workflowmodel/ingest/corpus/extra.json b/src/pkg/workflowmodel/ingest/corpus/extra.json new file mode 100644 index 0000000000..bb2cfcaab1 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/corpus/extra.json @@ -0,0 +1,8 @@ +[ + { + "name": "CNCF TechDocs", + "category": "non-landscape", + "repos": ["cncf/techdocs"], + "why": "CNCF TechDocs team repository; home of the AI-assisted documentation assessment workflow (curated model cncf-techdocs-assessment). Not a landscape project, so the survey never listed it." + } +] From e8ee5d862dc55f6155d20d8e7db6cb89e81cf97f Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:51:38 +0000 Subject: [PATCH 024/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20classify=20repeat?= =?UTF-8?q?ed=20agent-lifecycle=20log=20lines=20instead=20of=20filing=20un?= =?UTF-8?q?classified=20selfreports=20(#17757)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: classify repeated agent-lifecycle log lines instead of filing unclassified selfreports Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * test: expect agent_lifecycle_repeated class for non-held kick failures TestPostRestartKickHoldLogsWithoutFleetReport asserted the generic unclassified class; 'failed to send kick' is now classified as agent_lifecycle_repeated. Held kicks still produce no observation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/fixed-agent-lifecycle-repeated-class.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 6 ++++++ src/pkg/fleetreport/slogfeed/slogfeed_test.go | 2 +- 5 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-agent-lifecycle-repeated-class.md diff --git a/changelog.d/fixed-agent-lifecycle-repeated-class.md b/changelog.d/fixed-agent-lifecycle-repeated-class.md new file mode 100644 index 0000000000..fda13f7ad7 --- /dev/null +++ b/changelog.d/fixed-agent-lifecycle-repeated-class.md @@ -0,0 +1 @@ +- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index be45d81494..bd96ad2143 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -160,6 +160,7 @@ title is `[selfreport] : `. | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`), or a request the translator could not translate (`inference translate request failed`, logged at ERROR by `logTranslateFailure`), all in `crux/proxy/github_proxy_inference.go`, repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. A request refused because the agent has neither an agent model nor a gateway `default_model` logs `inference model not configured; request refused` at WARN instead and is not counted here (a benign configuration condition; see below). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (`SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller keeps serving the SHA without a message. A throttled 403/429 is treated as client-wide: every SHA is parked until `Retry-After` / `X-RateLimit-Reset` elapses and a single WARN is logged per hour per status (`warnGitHubFetchOnce`, key `sha-msg:throttled:`), so a rate limit or revoked token cannot trip this class. Other statuses (e.g. 404 for a force-pushed-away commit) back off per SHA and are deduped to once per hour per (SHA, status), so one stuck commit logs one line; the class is reported only when ≥5 distinct SHA/status pairs fail within 10 min (many branches whose commits GitHub no longer serves). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | +| `agent_lifecycle_repeated` | known agent-lifecycle log lines (`failed to send kick`, `agent CLI crashed (bare shell detected)`, `tmux kill-session before restart failed`, `agent tmux session missing`, watchdog `would restart dead agent` in observe mode) repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout`; replaces the catch-all `unclassified_error_repeated` for these lines | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low | | `kick_freshness_withheld` | a kick was withheld because every issue/PR ref failed the pre-kick freshness check (`kick withheld: every ref failed the freshness check`, `cmd/hive/kick_freshness.go`). The kick is retried on the next cycle; reported only when the same withholding repeats ≥5 times in 10 min (refs persistently stale or unreadable). Diagnostics count stale snapshots, closed items, deferred reads and other read errors for the last withheld kick, without including refs or raw errors. See local per-ref logs for details. Deferred API-budget refusals and refs the re-read found closed or merged log at INFO, so a kick withheld only by those never trips it. Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 0318cd1d7c..c0b7a6779d 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -92,6 +92,7 @@ const ( ClassLabelProvisioningFailed = "label_provisioning_failed" ClassAgentCLIRestartLoop = "agent_cli_restart_loop" ClassCopilotCLIHung = "copilot_cli_hung" + ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" ClassTranscriptScanSlow = "transcript_scan_slow" ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" ClassIssueClaimEffectFailed = "issue_claim_effect_failed" @@ -212,6 +213,7 @@ var Classes = map[string]ClassSpec{ ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, + ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index be1fa01910..1593918264 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -441,6 +441,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassIssueRequestStuck, message: "issue-request stuck: issue request failed 3 attempts over 7m"}, {class: ClassTmuxSessionCheckFailed, message: "tmux has-session failed"}, {class: ClassAgentPaneTimeout, message: "pane content at timeout"}, + {class: ClassAgentLifecycleRepeated, message: "agent tmux session missing"}, {class: ClassStartupKickInterrupted, message: "startup kick dropped: agent restarted or stopped while waiting"}, {class: ClassProxyTLSHandshakeFailed, message: "proxy client TLS handshake failed"}, {class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"}, @@ -1196,6 +1197,11 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { fed := map[string]string{ "startup kick dropped: agent restarted or stopped while waiting": ClassStartupKickInterrupted, "tmux has-session failed": ClassTmuxSessionCheckFailed, + "failed to send kick": ClassAgentLifecycleRepeated, + "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, + "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, + "agent tmux session missing": ClassAgentLifecycleRepeated, + "watchdog: would restart dead agent (observe mode; no action taken)": ClassAgentLifecycleRepeated, "pane content at timeout": ClassAgentPaneTimeout, "prompt timeout — dumping pane": ClassAgentPaneTimeout, "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index f49266b3de..4d2757e831 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -488,7 +488,7 @@ func TestPostRestartKickHoldLogsWithoutFleetReport(t *testing.T) { } // Other kick send failures must remain reportable. logger.Warn("failed to send kick", "agent", "reviewer-2", "error", "tmux send-keys exit 1") - if len(*seen) != 1 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated { + if len(*seen) != 1 || (*seen)[0].class != fleetreport.ClassAgentLifecycleRepeated { t.Fatalf("other kick failures must remain reportable: %+v", *seen) } } From 6c4620d087f87b8aa26b4e0ec50c5fc6e55796f4 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:02 +0000 Subject: [PATCH 025/223] =?UTF-8?q?=F0=9F=90=9B=20fix(marketplace):=20refu?= =?UTF-8?q?se=20and=20downgrade=20model-provenance=20curated=20claims=20fr?= =?UTF-8?q?om=20non-admin=20bundles=20(#17764)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(marketplace): refuse and downgrade model-provenance curated claims from non-admin bundles The hub checked only the envelope Tier when deciding whether a publisher may assert curated/built-in; the model's own spec.meta.provenance is hash-covered and stored verbatim. Spoke import preferred the model's provenance over the envelope tier, so a registered spoke could publish a bundle every importer badged as curated. The hub now refuses such bundles from non-admins (it cannot rewrite the hash-covered model) and importers honour a vouched model provenance only when the envelope asserts the same tier. Fixes #17763 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(marketplace): express importedProvenance as an if so the mutation gate sees its coverage Go's cover tool does not instrument the case expressions of a tagless switch, only the case bodies, so gremlins reported the three conditionals on the switch as NOT COVERED even though TestImportedProvenance exercises every branch. Fold the switch into a single if with identical semantics; if conditions sit inside the preceding coverage block and are attributed correctly. No behaviour change: empty provenance still takes the envelope tier, and a curated/built-in claim the envelope does not assert still falls back to it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../security-marketplace-model-provenance.md | 1 + .../dashboard/api_workflows_marketplace.go | 26 +++++++-- .../api_workflows_marketplace_test.go | 55 +++++++++++++++++++ src/pkg/hub/marketplace.go | 7 +++ src/pkg/hub/marketplace_test.go | 12 ++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 changelog.d/security-marketplace-model-provenance.md diff --git a/changelog.d/security-marketplace-model-provenance.md b/changelog.d/security-marketplace-model-provenance.md new file mode 100644 index 0000000000..2f6054f7a6 --- /dev/null +++ b/changelog.d/security-marketplace-model-provenance.md @@ -0,0 +1 @@ +- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go index b7836e8b2a..f9ffcdea91 100644 --- a/src/pkg/dashboard/api_workflows_marketplace.go +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -193,15 +193,29 @@ func (s *Server) importBundle(st *wfstore.Store, b *bundle.Bundle, by string) (w version = next } m.Spec.Meta.Version = version - if m.Spec.Meta.Provenance == "" { - m.Spec.Meta.Provenance = workflowmodel.Provenance(b.Tier) - } - if m.Spec.Meta.Provenance == workflowmodel.ProvenanceBuiltIn || string(m.Spec.Meta.Provenance) == bundle.TierLocalPublished { - m.Spec.Meta.Provenance = workflowmodel.ProvenanceLocal - } + m.Spec.Meta.Provenance = importedProvenance(m.Spec.Meta.Provenance, b.Tier) return st.Save(m, by, time.Now().UTC()) } +// importedProvenance decides what provenance an imported model is stored +// with. The envelope tier is the field the hub polices (curated/built-in +// need a hub admin); the model's own provenance is hash-covered and set by +// whoever authored the model, so a vouched tier claimed there is honoured +// only when the envelope asserts the same tier. built-in and +// local-published always land as local: this hive did not ship the model +// and has not published it. +func importedProvenance(model workflowmodel.Provenance, tier string) workflowmodel.Provenance { + p := model + vouched := p == workflowmodel.ProvenanceCurated || p == workflowmodel.ProvenanceBuiltIn + if p == "" || (vouched && string(p) != tier) { + p = workflowmodel.Provenance(tier) + } + if p == workflowmodel.ProvenanceBuiltIn || string(p) == bundle.TierLocalPublished || p == "" { + p = workflowmodel.ProvenanceLocal + } + return p +} + // hubRequest builds an authenticated spoke→hub request. func (s *Server) hubRequest(ctx context.Context, method, rawURL string, body []byte) (*http.Request, error) { var rd io.Reader diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go index eed2d9897b..735b3c0205 100644 --- a/src/pkg/dashboard/api_workflows_marketplace_test.go +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -458,3 +458,58 @@ func TestMarketplaceBaseFallsBackToHubURL(t *testing.T) { t.Fatal("server without deps should be enabled but have no base") } } + +func TestImportedProvenance(t *testing.T) { + cases := []struct { + model workflowmodel.Provenance + tier string + want workflowmodel.Provenance + }{ + // Empty model provenance takes the envelope tier. + {"", bundle.TierCurated, workflowmodel.ProvenanceCurated}, + {"", bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + {"", bundle.TierBuiltIn, workflowmodel.ProvenanceLocal}, + {"", "", workflowmodel.ProvenanceLocal}, + // A vouched tier the envelope agrees with is kept (built-in still lands as local). + {workflowmodel.ProvenanceCurated, bundle.TierCurated, workflowmodel.ProvenanceCurated}, + {workflowmodel.ProvenanceBuiltIn, bundle.TierBuiltIn, workflowmodel.ProvenanceLocal}, + // A vouched tier the envelope does not assert is laundering: fall back to the tier. + {workflowmodel.ProvenanceCurated, bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + {workflowmodel.ProvenanceCurated, bundle.TierIngestedUnreviewed, workflowmodel.ProvenanceIngestedUnreviewed}, + {workflowmodel.ProvenanceCurated, "", workflowmodel.ProvenanceLocal}, + {workflowmodel.ProvenanceBuiltIn, bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + // Lower tiers in the model are honoured as-is. + {workflowmodel.ProvenanceIngestedUnreviewed, bundle.TierLocalPublished, workflowmodel.ProvenanceIngestedUnreviewed}, + {workflowmodel.ProvenanceLocal, bundle.TierCurated, workflowmodel.ProvenanceLocal}, + } + for _, tc := range cases { + if got := importedProvenance(tc.model, tc.tier); got != tc.want { + t.Errorf("importedProvenance(%q, %q) = %q, want %q", tc.model, tc.tier, got, tc.want) + } + } +} + +// A bundle whose envelope the hub normalised to local-published but whose +// hash-covered model still claims curated must not import as curated. +func TestWorkflowImportDoesNotLaunderCuratedProvenance(t *testing.T) { + s := marketplaceServer(t, "") + m := techdocsModel(t) + m.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + b, err := bundle.Export(m, "spoke", time.Now()) + if err != nil { + t.Fatal(err) + } + b.Tier = bundle.TierLocalPublished + raw, _ := bundle.Encode(b) + rec := wfReq(s, http.MethodPost, "/api/workflows/import", string(raw), true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + got, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "1") + if err != nil { + t.Fatal(err) + } + if got.Spec.Meta.Provenance != workflowmodel.ProvenanceLocal { + t.Fatalf("laundered provenance = %q, want local", got.Spec.Meta.Provenance) + } +} diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 19750e96aa..0388838bb4 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -180,6 +180,13 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusBadRequest, "unknown tier "+b.Tier) return } + // The model's own provenance is hash-covered, so it cannot be normalised + // like the envelope tier; a vouched tier claimed there needs the same + // admin the envelope does, otherwise importers would store it as vouched. + if p := string(b.Model.Spec.Meta.Provenance); (p == bundle.TierCurated || p == bundle.TierBuiltIn) && !who.admin { + marketplaceJSONError(w, http.StatusForbidden, fmt.Sprintf("model provenance %q requires a hub admin", p)) + return + } if err := bundle.Verify(b); err != nil { marketplaceJSONError(w, http.StatusUnprocessableEntity, "bundle verification failed: "+err.Error()) return diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index c3b8e424de..d62f22c3f0 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -320,6 +320,18 @@ func TestMarketplacePublishAuth(t *testing.T) { if rec := mktDo(s, mktSpokeReq(mktBody(t, mktBundle(t, bundle.TierCurated)))); rec.Code != 403 { t.Fatalf("spoke curated: %d %s", rec.Code, rec.Body.String()) } + // Spoke asserting curated inside the hash-covered model (envelope tier + // legitimate) → 403: the hub cannot normalise it, so it must refuse. + launder := mktModel(t) + launder.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + lb, err := bundle.Export(launder, "spoke", time.Now()) + if err != nil { + t.Fatal(err) + } + lb.Tier = bundle.TierLocalPublished + if rec := mktDo(s, mktSpokeReq(mktBody(t, lb))); rec.Code != 403 || !strings.Contains(rec.Body.String(), "model provenance") { + t.Fatalf("spoke model-curated: %d %s", rec.Code, rec.Body.String()) + } // Spoke OK: `local` is normalised to local-published. rec := mktDo(s, mktSpokeReq(body)) if rec.Code != 201 { From 1971442025d173eda5f3676c947c46323f666c75 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:26 +0000 Subject: [PATCH 026/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20hub=20digest=20re?= =?UTF-8?q?ads=20a=20stale=20stage-tracker=20body=20instead=20of=20blockin?= =?UTF-8?q?g=20on=20a=20rebuild=20(#17785)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: let the hub activity digest read a stale stage-tracker body instead of blocking on a rebuild Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🧪 test: wait for the hub digest rebuild with testutil.Eventually instead of a sleep loop TestSleepRatchet failed on CI (196 time.Sleep calls in tests, baseline 195): the new stale-read test polled stageTrackerBuild with a hand-rolled time.Sleep loop. Use testutil.Eventually, which is the helper the ratchet points to. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-hub-digest-stale-tracker-read.md | 1 + src/pkg/dashboard/hub_activity.go | 2 +- src/pkg/dashboard/hub_activity_test.go | 54 +++++++++++++++++++ src/pkg/dashboard/stage_tracker_build.go | 28 +++++++--- 4 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 changelog.d/fixed-hub-digest-stale-tracker-read.md diff --git a/changelog.d/fixed-hub-digest-stale-tracker-read.md b/changelog.d/fixed-hub-digest-stale-tracker-read.md new file mode 100644 index 0000000000..12611376c5 --- /dev/null +++ b/changelog.d/fixed-hub-digest-stale-tracker-read.md @@ -0,0 +1 @@ +- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. diff --git a/src/pkg/dashboard/hub_activity.go b/src/pkg/dashboard/hub_activity.go index a278242a58..0010590830 100644 --- a/src/pkg/dashboard/hub_activity.go +++ b/src/pkg/dashboard/hub_activity.go @@ -63,7 +63,7 @@ func (s *Server) HubActivitySnapshot(ctx context.Context) (hubactivity.Snapshot, if status == nil { return hubactivity.Snapshot{}, false } - body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false) + body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false, true) if body == nil { return hubactivity.Snapshot{}, false } diff --git a/src/pkg/dashboard/hub_activity_test.go b/src/pkg/dashboard/hub_activity_test.go index ea8f730c73..206ecc34a4 100644 --- a/src/pkg/dashboard/hub_activity_test.go +++ b/src/pkg/dashboard/hub_activity_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/hivecommons/hive/internal/testutil" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/hubactivity" "github.com/hivecommons/hive/pkg/stagelabels" @@ -110,3 +111,56 @@ func TestOnStageTrackerPublishFiresOnInvalidation(t *testing.T) { var nilServer *Server nilServer.OnStageTrackerPublish(func() {}) } + +// TestHubActivityItemsServesStaleBodyWithoutBuilding pins +// hivecommons/hive#17776: once the tracker has a body, an invalidated cache +// must not make the digest read wait on a rebuild. +func TestHubActivityItemsServesStaleBodyWithoutBuilding(t *testing.T) { + t.Setenv("HIVE_PUBLIC_REPOS", "") + now := time.Date(2026, 10, 31, 9, 0, 0, 0, time.UTC) + s := newTestServer() + s.stageStore = stagelabels.NewStore("", func() time.Time { return now }) + s.statusMu.Lock() + s.status = minimalPayload() + s.statusMu.Unlock() + + release := make(chan struct{}) + started := make(chan struct{}, 2) + s.stageTrackerBuildFn = func(_ *StatusPayload, _ *stagelabels.Store, at time.Time) StageTrackerResponse { + started <- struct{}{} + <-release + return StageTrackerResponse{GeneratedAt: at, Items: []StageTrackerItem{{Repo: "acme/widget", Number: 7, Kind: "pr", Modifiers: []string{}}}} + } + close(release) + if _, ok := s.HubActivitySnapshot(context.Background()); !ok { + t.Fatal("first read must build a body when none exists") + } + <-started + + release = make(chan struct{}) + s.invalidateStageTrackerCache() + type result struct { + snap hubactivity.Snapshot + ok bool + } + done := make(chan result, 1) + go func() { + snap, ok := s.HubActivitySnapshot(context.Background()) + done <- result{snap, ok} + }() + select { + case r := <-done: + if !r.ok || len(r.snap.Items) != 1 { + t.Fatalf("stale read = %+v, %v; want the previous body", r.snap, r.ok) + } + case <-time.After(2 * time.Second): + t.Fatal("digest read blocked on the rebuild instead of serving the stale body") + } + <-started + close(release) + testutil.Eventually(t, 2*time.Second, func() bool { + s.stageTrackerCacheMu.Lock() + defer s.stageTrackerCacheMu.Unlock() + return s.stageTrackerBuild == nil + }, "background rebuild never finished") +} diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index 2a99c9406e..b6f57dcefd 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -887,13 +887,16 @@ func (s *Server) stageTrackerPayload(status *StatusPayload, now time.Time) (body // came from (stageTrackerSource*Name; "" with a nil body when the build // failed or ctx ended while waiting on an in-flight build). func (s *Server) stageTrackerPayloadSource(ctx context.Context, status *StatusPayload, now time.Time) (body []byte, etag string, generatedAt time.Time, source string) { - return s.stageTrackerPayloadFrom(ctx, status, now, true) + return s.stageTrackerPayloadFrom(ctx, status, now, true, false) } // stageTrackerPayloadFrom is stageTrackerPayloadSource; markServed false // reads the tracker without counting as a client that asked for it, so an -// in-process reader (the hub activity digest) adds no SSE nudges. -func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayload, now time.Time, markServed bool) (body []byte, etag string, generatedAt time.Time, source string) { +// in-process reader (the hub activity digest) adds no SSE nudges. preferStale +// serves a stale body at once when no build is in flight and rebuilds in the +// background, so a latency-bound reader is never the synchronous builder +// (hivecommons/hive#17776); with no body at all it still builds in place. +func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayload, now time.Time, markServed, preferStale bool) (body []byte, etag string, generatedAt time.Time, source string) { ttl := s.stageTrackerCacheTTL() s.stageTrackerCacheMu.Lock() if markServed { @@ -927,6 +930,20 @@ func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayl gen := s.stageTrackerGen s.stageTrackerCacheMu.Unlock() + if preferStale && len(c.body) > 0 { + go s.finishStageTrackerBuild(inflight, status, now, ttl, gen) + return c.body, c.etag, c.generatedAt, stageTrackerSourceStaleName + } + built, ok := s.finishStageTrackerBuild(inflight, status, now, ttl, gen) + if !ok { + return nil, "", now, "" + } + return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName +} + +// finishStageTrackerBuild runs the claimed in-flight build, stores the +// result and releases waiters. +func (s *Server) finishStageTrackerBuild(inflight *stageTrackerBuild, status *StatusPayload, now time.Time, ttl time.Duration, gen uint64) (stageTrackerCache, bool) { built, ok := s.encodeStageTracker(status, now, ttl) s.stageTrackerCacheMu.Lock() @@ -939,10 +956,7 @@ func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayl s.stageTrackerBuild = nil close(inflight.done) s.stageTrackerCacheMu.Unlock() - if !ok { - return nil, "", now, "" - } - return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName + return built, ok } // encodeStageTracker builds and encodes the tracker for status. It holds no From 29d8c427ca7683a6627edf92555dfadb4d55919b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:48 +0000 Subject: [PATCH 027/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20log=20issue-claim?= =?UTF-8?q?s=20GitHub=20side-effect=20retries=20below=20WARN=20(#17786)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-issue-claims-retry-warn-dedupe.md | 1 + src/cmd/hive/claimswire.go | 2 +- src/cmd/hive/claimswire_retry_test.go | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-issue-claims-retry-warn-dedupe.md diff --git a/changelog.d/fixed-issue-claims-retry-warn-dedupe.md b/changelog.d/fixed-issue-claims-retry-warn-dedupe.md new file mode 100644 index 0000000000..0942f81c03 --- /dev/null +++ b/changelog.d/fixed-issue-claims-retry-warn-dedupe.md @@ -0,0 +1 @@ +- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). diff --git a/src/cmd/hive/claimswire.go b/src/cmd/hive/claimswire.go index 58b15080e5..ddb4cdd1f2 100644 --- a/src/cmd/hive/claimswire.go +++ b/src/cmd/hive/claimswire.go @@ -93,7 +93,7 @@ func retryClaimEffect(ctx context.Context, logger *slog.Logger, what, key string return n } if logger != nil { - logger.Warn("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) + logger.Info("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) } if !wait(ctx, backoff) { if logger != nil { diff --git a/src/cmd/hive/claimswire_retry_test.go b/src/cmd/hive/claimswire_retry_test.go index 3792af3928..b3ac7d9559 100644 --- a/src/cmd/hive/claimswire_retry_test.go +++ b/src/cmd/hive/claimswire_retry_test.go @@ -1,9 +1,12 @@ package main import ( + "bytes" "context" "errors" + "log/slog" "net/http" + "strings" "testing" "time" @@ -138,3 +141,23 @@ func TestClaimEffectRetryable(t *testing.T) { } } } + +// An expected transient retry is logged below WARN so the fleet-report +// classifier does not file it; only exhaustion warns, once +// (hivecommons/hive#17782). +func TestRetryClaimEffect_RetryLoggedBelowWarn(t *testing.T) { + swapClaimsEffectRetry(t, 3, nil) + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + retryClaimEffect(context.Background(), logger, "claimed stage", "acme/widgets#7", func() error { + return mutation.ErrNeedsReconciliation + }) + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + if strings.Contains(line, "retrying GitHub side effect") && strings.Contains(line, "level=WARN") { + t.Fatalf("retry logged at WARN: %s", line) + } + } + if got := strings.Count(buf.String(), "GitHub side effect failed"); got != 1 { + t.Fatalf("exhaustion warnings = %d, want 1", got) + } +} From c101c91802f43c86387b315b4a68f6524b58509a Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:54:20 +0000 Subject: [PATCH 028/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20detect=20orphaned?= =?UTF-8?q?=20queued=20workflow=20runs=20in=20automerge=20held-run=20resol?= =?UTF-8?q?ution=20(#17787)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-automerge-orphaned-queued-runs.md | 1 + .../github/automerge/automerge_held_runs.go | 46 +++++++++++++++- .../automerge/automerge_held_runs_test.go | 55 +++++++++++++++++++ .../github/automerge/automerge_sweep_test.go | 14 +++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-automerge-orphaned-queued-runs.md diff --git a/changelog.d/fixed-automerge-orphaned-queued-runs.md b/changelog.d/fixed-automerge-orphaned-queued-runs.md new file mode 100644 index 0000000000..6e7f72b821 --- /dev/null +++ b/changelog.d/fixed-automerge-orphaned-queued-runs.md @@ -0,0 +1 @@ +- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). diff --git a/src/pkg/github/automerge/automerge_held_runs.go b/src/pkg/github/automerge/automerge_held_runs.go index 3f871225f5..c09935e9f0 100644 --- a/src/pkg/github/automerge/automerge_held_runs.go +++ b/src/pkg/github/automerge/automerge_held_runs.go @@ -115,13 +115,19 @@ const ( // is still treated as "GitHub has not scheduled them yet". heldRunNeverStartedGrace = 30 * time.Minute workflowRunsPageSize = 100 + // workflowRunStatusQueued is the status of a run GitHub has not started. + workflowRunStatusQueued = "queued" + // orphanedRunWarnKey keys the once-per-head orphaned-run warning in + // terminalHoldLogged. + orphanedRunWarnKey = "orphaned-queued-runs:" ) // resolveMissingRequiredCheck refines a `required-check-missing:` skip // for a trusted-lane PR. It returns the original reason when nothing is known, // a workflow-approval-required reason when GitHub is waiting for approval and // Hive has posted `/ok-to-test`, or a never-started/never-ran reason when -// polling cannot progress. It never approves a run with the App token. +// polling cannot progress. Runs GitHub orphaned in `queued` with zero jobs +// count as never started (#17775). It never approves a run with the App token. func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, owner, repo string, pr *gh.PullRequest, number int, headSHA, reason string, headPushedAt time.Time) string { if c == nil || c.gh == nil || !strings.HasPrefix(reason, requiredCheckMissingPrefix) { return reason @@ -141,6 +147,14 @@ func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, o } held := heldSameRepoWorkflowRuns(runs.WorkflowRuns, owner+"/"+repo) if len(held) == 0 { + startedRuns := len(runs.WorkflowRuns) + if orphaned := c.orphanedQueuedWorkflowRuns(ctx, owner, repo, runs.WorkflowRuns); orphaned > 0 && orphaned == startedRuns { + startedRuns = 0 + if c.markTerminalHoldLogged(owner, repo, number, headSHA, orphanedRunWarnKey+missing) { + c.warn("automerge workflow runs for head are orphaned in queued with no jobs; treating them as never started", + "repo", displayRepo, "pr", number, "sha", headSHA, "orphaned", orphaned, "context", missing) + } + } if c.rebaseNudged(owner, repo, number, headSHA) || c.botTakeoverAttempted(owner, repo, number, headSHA) { return requiredCheckNeverRanPrefix + missing } @@ -148,10 +162,10 @@ func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, o c.nudgeDependabotRebase(ctx, displayRepo, owner, repo, number, headSHA, missing) return requiredCheckNeverRanPrefix + missing } - if c.botTakeoverEligible(pr, len(runs.WorkflowRuns), headPushedAt) { + if c.botTakeoverEligible(pr, startedRuns, headPushedAt) { return c.takeOverBotPR(ctx, displayRepo, owner, repo, pr, number, headSHA, missing) } - if len(runs.WorkflowRuns) == 0 && headOlderThan(c.now, headPushedAt, heldRunNeverStartedGrace) { + if startedRuns == 0 && headOlderThan(c.now, headPushedAt, heldRunNeverStartedGrace) { return requiredCheckNeverStartedPrefix + missing } return reason @@ -582,6 +596,32 @@ func heldSameRepoWorkflowRuns(runs []*gh.WorkflowRun, fullName string) []*gh.Wor return held } +// orphanedQueuedWorkflowRuns counts same-repo runs GitHub created but left in +// `queued` past heldRunNeverStartedGrace without materializing a single job. +// Such runs never progress, so they are no evidence CI is coming (#17775). +func (c *Engine) orphanedQueuedWorkflowRuns(ctx context.Context, owner, repo string, runs []*gh.WorkflowRun) int { + orphaned := 0 + for _, run := range runs { + if run == nil || !strings.EqualFold(run.GetStatus(), workflowRunStatusQueued) { + continue + } + if run.GetHeadRepository() == nil || !strings.EqualFold(run.GetHeadRepository().GetFullName(), owner+"/"+repo) { + continue + } + if !headOlderThan(c.now, run.GetCreatedAt().Time, heldRunNeverStartedGrace) { + continue + } + jobs, _, err := c.gh.Actions.ListWorkflowJobs(hgithub.WithRESTCaller(ctx, "hive:automerge_sweep"), owner, repo, run.GetID(), &gh.ListWorkflowJobsOptions{ + ListOptions: gh.ListOptions{PerPage: 1}, + }) + if err != nil || jobs == nil || jobs.GetTotalCount() > 0 || len(jobs.Jobs) > 0 { + continue + } + orphaned++ + } + return orphaned +} + func headOlderThan(now func() time.Time, headPushedAt time.Time, grace time.Duration) bool { if headPushedAt.IsZero() { return false diff --git a/src/pkg/github/automerge/automerge_held_runs_test.go b/src/pkg/github/automerge/automerge_held_runs_test.go index 2cf239b1ff..f713c241f0 100644 --- a/src/pkg/github/automerge/automerge_held_runs_test.go +++ b/src/pkg/github/automerge/automerge_held_runs_test.go @@ -131,6 +131,61 @@ func TestTrySweepSelfAuthoredPRNamesNeverStartedRequiredCheck(t *testing.T) { } } +// TestTrySweepSelfAuthoredPRTreatsOrphanedQueuedRunsAsNeverStarted pins +// #17775: GitHub can leave a head's runs `queued` with zero jobs forever, and +// such runs must not keep the PR at a bare required-check-missing reason. +func TestTrySweepSelfAuthoredPRTreatsOrphanedQueuedRunsAsNeverStarted(t *testing.T) { + now := time.Date(2026, 11, 11, 22, 10, 0, 0, time.UTC) + old := now.Add(-2 * heldRunNeverStartedGrace).Format(time.RFC3339) + fresh := now.Add(-time.Minute).Format(time.RFC3339) + queuedRun := func(id int, headRepo, createdAt string) map[string]any { + return map[string]any{ + "id": id, "name": "v2 CI", "status": "queued", "created_at": createdAt, + "head_repository": map[string]any{"full_name": headRepo}, + } + } + for _, tc := range []struct { + name string + runs []map[string]any + jobs map[int]int + want string + }{ + {name: "old queued runs with zero jobs", runs: []map[string]any{queuedRun(1, "acme/widget", old), queuedRun(2, "acme/widget", old)}, want: requiredCheckNeverStartedPrefix + "ci/build"}, + {name: "queued run with jobs", runs: []map[string]any{queuedRun(1, "acme/widget", old)}, jobs: map[int]int{1: 1}, want: "required-check-missing:ci/build"}, + {name: "freshly queued run", runs: []map[string]any{queuedRun(1, "acme/widget", fresh)}, want: "required-check-missing:ci/build"}, + {name: "fork head queued run", runs: []map[string]any{queuedRun(1, "mallory/widget", old)}, want: "required-check-missing:ci/build"}, + { + name: "orphaned run beside an in-progress run", + runs: []map[string]any{queuedRun(1, "acme/widget", old), {"id": 2, "name": "Tests", "status": "in_progress", "head_repository": map[string]any{"full_name": "acme/widget"}}}, + want: "required-check-missing:ci/build", + }, + } { + t.Run(tc.name, func(t *testing.T) { + pr := missingRequiredPR(func(p *selfAuthoredPR) { + p.workflowRuns = tc.runs + p.workflowRunJobs = tc.jobs + p.updatedAt = old + }) + var merged []int + api := newSelfAuthoredAutoMergeAPI(t, []selfAuthoredPR{pr}, &merged) + defer api.Close() + c := newAutoMergeSweepClient(api.URL) + c.now = func() time.Time { return now } + + _, reason, err := c.trySweepSelfAuthoredPR(context.Background(), "widget", "acme", "widget", 23, true) + if err != nil { + t.Fatalf("trySweepSelfAuthoredPR returned error: %v", err) + } + if reason != tc.want { + t.Fatalf("reason = %q, want %q", reason, tc.want) + } + if len(merged) != 0 { + t.Fatalf("merged = %v, want no merge", merged) + } + }) + } +} + func TestTerminalRequiredCheckHoldLoggedOncePerHeadAndReason(t *testing.T) { tests := []struct { name string diff --git a/src/pkg/github/automerge/automerge_sweep_test.go b/src/pkg/github/automerge/automerge_sweep_test.go index 408fbc68e6..a139bcc2f9 100644 --- a/src/pkg/github/automerge/automerge_sweep_test.go +++ b/src/pkg/github/automerge/automerge_sweep_test.go @@ -1822,6 +1822,9 @@ type selfAuthoredPR struct { updatedAt string // workflowRuns are returned by GET /actions/runs?head_sha=. workflowRuns []map[string]any + // workflowRunJobs is the job count GET /actions/runs/{id}/jobs reports + // per run id (absent = zero jobs). + workflowRunJobs map[int]int // headCommitAuthor is the login GET /commits/{head} reports as author; // commentBodies collects POST /issues/{n}/comments bodies. headCommitAuthor string @@ -1898,6 +1901,17 @@ func newSelfAuthoredAutoMergeAPI(t *testing.T, prs []selfAuthoredPR, merged *[]i runs = []map[string]any{} } json.NewEncoder(w).Encode(map[string]any{"total_count": len(runs), "workflow_runs": runs}) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/actions/runs/") && strings.HasSuffix(r.URL.Path, "/jobs"): + id := pathNumber(t, r.URL.Path, "/repos/acme/widget/actions/runs/", "/jobs") + count := 0 + for _, pr := range prs { + count += pr.workflowRunJobs[id] + } + jobs := []map[string]any{} + for i := 0; i < count; i++ { + jobs = append(jobs, map[string]any{"id": i + 1, "run_id": id, "status": "queued"}) + } + json.NewEncoder(w).Encode(map[string]any{"total_count": count, "jobs": jobs}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main": json.NewEncoder(w).Encode(map[string]any{"commit": map[string]string{"sha": "basesha"}}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main/protection/required_status_checks": From 67efaf34c3db6d200690d17986e06febd4465394 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:54:43 +0000 Subject: [PATCH 029/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20surface=20agen?= =?UTF-8?q?t=5Fsandbox.substrate=20and=20per-agent=20sandbox.substrate=20s?= =?UTF-8?q?ettings=20(#17788)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-settings-substrate-leaves.md | 1 + src/pkg/dashboard/api_agent_config.go | 29 +++++ src/pkg/dashboard/api_governor_security.go | 101 ++++++++++++++---- .../dashboard/settings_parity_allowlist.txt | 38 ------- .../dashboard/static-src/js/config-agent.js | 3 + .../static-src/js/config-watchdog-security.js | 66 ++++++++++++ src/pkg/dashboard/static/index.html | 69 ++++++++++++ 7 files changed, 248 insertions(+), 59 deletions(-) create mode 100644 changelog.d/added-settings-substrate-leaves.md diff --git a/changelog.d/added-settings-substrate-leaves.md b/changelog.d/added-settings-substrate-leaves.md new file mode 100644 index 0000000000..59d543165c --- /dev/null +++ b/changelog.d/added-settings-substrate-leaves.md @@ -0,0 +1 @@ +- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. diff --git a/src/pkg/dashboard/api_agent_config.go b/src/pkg/dashboard/api_agent_config.go index 081f6a677f..17ab41cf89 100644 --- a/src/pkg/dashboard/api_agent_config.go +++ b/src/pkg/dashboard/api_agent_config.go @@ -184,6 +184,7 @@ func (s *Server) handleAgentConfigGet(w http.ResponseWriter, r *http.Request) { "jevMode": agentCfg.JevMode, "jevReady": s.deps.Config.JevReady(), "explainMode": agentCfg.ExplainMode, + "sandboxSubstrate": agentSandboxSubstrateOrEmpty(agentCfg), "sandboxEnabled": agentCfg.Sandbox != nil && agentCfg.Sandbox.Enabled != nil && *agentCfg.Sandbox.Enabled, "sandboxEffective": agentCfg.SandboxEnabled(s.deps.Config.AgentSandbox), "replicas": agentCfg.Replicas, @@ -661,6 +662,27 @@ func (s *Server) handleAgentConfigGeneral(w http.ResponseWriter, r *http.Request agentCfg.Sandbox.Enabled = &bb } } + if v, ok := body["sandboxSubstrate"]; ok { + raw, err := json.Marshal(v) + var in config.SandboxSubstrateConfig + if err == nil { + err = json.Unmarshal(raw, &in) + } + var sub *config.SandboxSubstrateConfig + if err == nil { + sub, err = sanitizeSubstrateConfig(in) + } + if err != nil { + jsonError(w, "invalid sandboxSubstrate: "+err.Error(), http.StatusBadRequest) + return + } + if sub != nil && agentCfg.Sandbox == nil { + agentCfg.Sandbox = &config.AgentSandboxOverride{} + } + if agentCfg.Sandbox != nil { + agentCfg.Sandbox.Substrate = sub + } + } if v, ok := body["cavemanMode"]; ok { if s, ok := v.(string); ok { s = sanitizeString(s) @@ -1915,3 +1937,10 @@ func (s *Server) getAgentHooks(name string) map[string][]any { } return map[string][]any{"preKick": {}, "postIdle": {}} } + +func agentSandboxSubstrateOrEmpty(a config.AgentConfig) config.SandboxSubstrateConfig { + if a.Sandbox == nil { + return config.SandboxSubstrateConfig{} + } + return substrateOrEmpty(a.Sandbox.Substrate) +} diff --git a/src/pkg/dashboard/api_governor_security.go b/src/pkg/dashboard/api_governor_security.go index 85b3928712..99889a117c 100644 --- a/src/pkg/dashboard/api_governor_security.go +++ b/src/pkg/dashboard/api_governor_security.go @@ -1,8 +1,10 @@ package dashboard import ( + "errors" "net/http" "os" + "reflect" "strings" "github.com/hivecommons/hive/pkg/config" @@ -33,27 +35,28 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) IntentAlignmentModel *string `json:"intentAlignmentModel"` // Intent classifier lists. An empty list restores the built-in // defaults (nil), matching how the intent package treats them. - IntentTestPathPatterns *[]string `json:"intentTestPathPatterns"` - IntentDocsPathPatterns *[]string `json:"intentDocsPathPatterns"` - IntentGuardrailPathPatterns *[]string `json:"intentGuardrailPathPatterns"` - IntentFeatureSignals *[]string `json:"intentFeatureSignals"` - ReviewRequire *bool `json:"reviewRequireApproval"` - ReviewFanOut *bool `json:"reviewFanOut"` - ReviewMaxParallel *int `json:"reviewMaxParallelReviews"` - ReviewReviewerAgents *[]string `json:"reviewReviewerAgents"` - ReviewFixerAgent *string `json:"reviewFixerAgent"` - AgentSandboxEnabled *bool `json:"agentSandboxEnabled"` - SandboxImage *string `json:"agentSandboxImage"` - SandboxEnvAllowlist *[]string `json:"agentSandboxEnvAllowlist"` - SandboxNetworkMode *string `json:"agentSandboxNetworkMode"` - SandboxTimeoutS *int `json:"agentSandboxTimeoutS"` - SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` - SandboxRuntime *string `json:"agentSandboxRuntime"` - SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` - SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` - SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` - SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` - SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` + IntentTestPathPatterns *[]string `json:"intentTestPathPatterns"` + IntentDocsPathPatterns *[]string `json:"intentDocsPathPatterns"` + IntentGuardrailPathPatterns *[]string `json:"intentGuardrailPathPatterns"` + IntentFeatureSignals *[]string `json:"intentFeatureSignals"` + ReviewRequire *bool `json:"reviewRequireApproval"` + ReviewFanOut *bool `json:"reviewFanOut"` + ReviewMaxParallel *int `json:"reviewMaxParallelReviews"` + ReviewReviewerAgents *[]string `json:"reviewReviewerAgents"` + ReviewFixerAgent *string `json:"reviewFixerAgent"` + AgentSandboxEnabled *bool `json:"agentSandboxEnabled"` + SandboxImage *string `json:"agentSandboxImage"` + SandboxEnvAllowlist *[]string `json:"agentSandboxEnvAllowlist"` + SandboxNetworkMode *string `json:"agentSandboxNetworkMode"` + SandboxTimeoutS *int `json:"agentSandboxTimeoutS"` + SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` + SandboxRuntime *string `json:"agentSandboxRuntime"` + SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` + SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` + SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` + SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` + SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` + SandboxSubstrate *config.SandboxSubstrateConfig `json:"agentSandboxSubstrate"` // Sentinel (suspicious-activity alerts) is pointer-typed per field // so an absent key means "unchanged" — a save that only toggled a // behavior must not wipe the sensitive-path list. @@ -263,6 +266,14 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) job.TTLSeconds = *body.SandboxJobTTLSeconds } } + if body.SandboxSubstrate != nil { + sub, err := sanitizeSubstrateConfig(*body.SandboxSubstrate) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + cfg.AgentSandbox.Substrate = sub + } if nextSentinel != nil { cfg.Sentinel = *nextSentinel } @@ -361,6 +372,7 @@ func securitySectionResponse(cfg *config.Config) map[string]interface{} { "agentSandboxJobTtlSeconds": sandboxJobOrEmpty(cfg).TTLSeconds, "agentSandboxJobWorkspaceClaim": sandboxJobOrEmpty(cfg).WorkspaceClaim, "agentSandboxJobWorkspaceClaimMount": sandboxJobOrEmpty(cfg).WorkspaceClaimMount, + "agentSandboxSubstrate": substrateOrEmpty(cfg.AgentSandbox.Substrate), "sandboxedAgents": sandboxed, "totalAgents": len(cfg.Agents), "sandboxWarnings": sandboxWarnings, @@ -458,3 +470,50 @@ func sandboxJobOrEmpty(cfg *config.Config) config.SandboxJobConfig { } return *cfg.AgentSandbox.Job } + +func substrateOrEmpty(sub *config.SandboxSubstrateConfig) config.SandboxSubstrateConfig { + if sub == nil { + return config.SandboxSubstrateConfig{} + } + return *sub +} + +// sanitizeSubstrateConfig trims and validates a Substrate block from the +// Settings dialog. An all-empty block returns nil so the YAML key is dropped. +func sanitizeSubstrateConfig(in config.SandboxSubstrateConfig) (*config.SandboxSubstrateConfig, error) { + if in.TimeoutS < 0 || in.BridgePort < 0 || in.CPUMillicores < 0 || in.MemoryBytes < 0 { + return nil, errors.New("substrate numeric settings must be >= 0") + } + clean := func(s string) string { return sanitizeString(strings.TrimSpace(s)) } + out := config.SandboxSubstrateConfig{ + Endpoint: clean(in.Endpoint), + Atespace: clean(in.Atespace), + TokenEnv: clean(in.TokenEnv), + TokenFile: clean(in.TokenFile), + CAFile: clean(in.CAFile), + InsecureTLS: in.InsecureTLS, + TimeoutS: in.TimeoutS, + RouterURL: clean(in.RouterURL), + RouterClientBundle: clean(in.RouterClientBundle), + RouterCAFile: clean(in.RouterCAFile), + BridgePort: in.BridgePort, + Image: clean(in.Image), + BridgeCommand: sanitizeStringSlice(in.BridgeCommand), + CPUMillicores: in.CPUMillicores, + MemoryBytes: in.MemoryBytes, + WorkerSelector: sanitizedHeaderMap(in.WorkerSelector), + SnapshotOnIdle: in.SnapshotOnIdle, + AllowHosts: sanitizeStringSlice(in.AllowHosts), + PolicyProfile: clean(in.PolicyProfile), + } + if len(out.BridgeCommand) == 0 { + out.BridgeCommand = nil + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = nil + } + if reflect.DeepEqual(out, config.SandboxSubstrateConfig{}) { + return nil, nil + } + return &out, nil +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index eeee612fb8..718fbbf15f 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -29,25 +29,6 @@ agent_sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell runtime; agent_sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) -agent_sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agent_sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) agent_sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..acmm_levels[] # TODO #11854 @@ -166,25 +147,6 @@ agents..sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell r agents..sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) -agents..sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) diff --git a/src/pkg/dashboard/static-src/js/config-agent.js b/src/pkg/dashboard/static-src/js/config-agent.js index f4cfd4b815..ef85124de9 100644 --- a/src/pkg/dashboard/static-src/js/config-agent.js +++ b/src/pkg/dashboard/static-src/js/config-agent.js @@ -123,6 +123,9 @@
Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''} +
Substrate sandbox overrides + ${substrateSandboxFields('general', g.sandboxSubstrate)} +
Clear on Kick ${settingHelpMark('agent-configuration', 'anatomy-of-an-agent', 'Clear on Kick', `Send /clear before each governor kick to reset conversation context. Prevents stale context buildup across runs. On by default for all agents.`)} diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index 3a74632df6..8ac65300be 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -526,6 +526,69 @@ markDirty('security', key, n); } + const SUBSTRATE_SANDBOX_FIELDS = [ + { key: 'endpoint', label: 'Endpoint', kind: 'text', global: 'agent_sandbox.substrate.endpoint', agent: 'agents..sandbox.substrate.endpoint', help: `Substrate Control API address (host:port or an http(s):// URL). Empty uses api.ate-system.svc:443.` }, + { key: 'atespace', label: 'Atespace', kind: 'text', global: 'agent_sandbox.substrate.atespace', agent: 'agents..sandbox.substrate.atespace', help: `Substrate atespace (its tenant namespace, not a Kubernetes namespace) actors are created in. Empty uses hive.` }, + { key: 'token_env', label: 'Token env', kind: 'text', global: 'agent_sandbox.substrate.token_env', agent: 'agents..sandbox.substrate.token_env', help: `Environment variable holding a Control API bearer token. Empty reads the token file instead.` }, + { key: 'token_file', label: 'Token file', kind: 'text', global: 'agent_sandbox.substrate.token_file', agent: 'agents..sandbox.substrate.token_file', help: `Projected ServiceAccount token file hive authenticates with. Empty uses /var/run/secrets/ate/token.` }, + { key: 'ca_file', label: 'CA file', kind: 'text', global: 'agent_sandbox.substrate.ca_file', agent: 'agents..sandbox.substrate.ca_file', help: `PEM bundle the Control API serving certificate chains to. Empty uses the projected CA when present, else the system roots.` }, + { key: 'insecure_tls', label: 'Insecure TLS', kind: 'bool', global: 'agent_sandbox.substrate.insecure_tls', agent: 'agents..sandbox.substrate.insecure_tls', help: `Skip Control API and router certificate verification. Dev only.` }, + { key: 'timeout_s', label: 'Timeout (seconds)', kind: 'int', global: 'agent_sandbox.substrate.timeout_s', agent: 'agents..sandbox.substrate.timeout_s', help: `Bounds one actor from create to delete. 0 uses the sandbox timeout.` }, + { key: 'router_url', label: 'Router URL', kind: 'text', global: 'agent_sandbox.substrate.router_url', agent: 'agents..sandbox.substrate.router_url', help: `atenet-router CONNECT listener hive reaches the actor bridge through. Empty uses the in-cluster default.` }, + { key: 'router_client_bundle', label: 'Router client bundle', kind: 'text', global: 'agent_sandbox.substrate.router_client_bundle', agent: 'agents..sandbox.substrate.router_client_bundle', help: `podCertificate bundle the router authenticates hive with. Empty uses the default path.` }, + { key: 'router_ca_file', label: 'Router CA file', kind: 'text', global: 'agent_sandbox.substrate.router_ca_file', agent: 'agents..sandbox.substrate.router_ca_file', help: `CA the router serving certificate chains to. Empty uses the default path.` }, + { key: 'bridge_port', label: 'Bridge port', kind: 'int', global: 'agent_sandbox.substrate.bridge_port', agent: 'agents..sandbox.substrate.bridge_port', help: `Port the in-actor bridge listens on. 0 uses 8477.` }, + { key: 'image', label: 'Image', kind: 'text', global: 'agent_sandbox.substrate.image', agent: 'agents..sandbox.substrate.image', help: `Digest-pinned sandbox image (…@sha256:…) actors run. Substrate refuses tag-only references.` }, + { key: 'bridge_command', label: 'Bridge command', kind: 'lines', global: 'agent_sandbox.substrate.bridge_command[]', agent: 'agents..sandbox.substrate.bridge_command[]', help: `Actor entrypoint that serves the bridge. One argument per line; empty uses the crux substrate-bridge subcommand.` }, + { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, + { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, + { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, + { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, + ]; + + /* substrateSandboxFields renders the Substrate runtime block for either the + global agent_sandbox.substrate (section 'security') or one agent's + sandbox.substrate override (section 'general'). */ + function substrateSandboxFields(section, sub) { + const dirtyKey = section === 'security' ? 'agentSandboxSubstrate' : 'sandboxSubstrate'; + sub = sub || {}; + return SUBSTRATE_SANDBOX_FIELDS.map(f => { + const yaml = section === 'security' ? f.global : f.agent; + const label = 'Substrate ' + f.label.toLowerCase(); + const head = '
'; + const attrs = 'data-arg0="' + section + '" data-arg1="' + dirtyKey + '" data-arg2="' + f.key + '" data-arg3="' + f.kind + '"'; + const v = sub[f.key]; + if (f.kind === 'bool') { + return head + '
'; + } + if (f.kind === 'int') { + return head + '
'; + } + if (f.kind === 'lines' || f.kind === 'map') { + const text = f.kind === 'map' + ? Object.keys(v || {}).sort().map(k => k + '=' + v[k]).join('\n') + : (v || []).join('\n'); + return head + ''; + } + return head + ''; + }).join('\n'); + } + + function markSubstrateSandbox(section, dirtyKey, field, kind, value) { + const pending = _configState.dirty[section] && _configState.dirty[section][dirtyKey]; + const base = pending || ((_configState.data && _configState.data[section]) || {})[dirtyKey] || {}; + const next = Object.assign({}, base); + if (kind === 'bool') next[field] = !!value; + else if (kind === 'int') next[field] = Math.max(0, Math.floor(Number(value) || 0)); + else if (kind === 'lines') next[field] = String(value || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean); + else if (kind === 'map') next[field] = parseSettingsMap(value); + else next[field] = String(value || '').trim(); + if (_configState.data && _configState.data[section]) _configState.data[section][dirtyKey] = next; + markDirty(section, dirtyKey, next); + } + function parseSecurityList(value) { return String(value || '').split(',').map(s => s.trim()).filter(Boolean); } @@ -927,6 +990,9 @@
+
Substrate sandbox + ${substrateSandboxFields('security', sec.agentSandboxSubstrate)} +
`; /* Backup encryption key: painted asynchronously from presence-only diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 92bfc42a08..68c2d1d5ce 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -59612,6 +59612,9 @@

Hub settings

Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''} +
Substrate sandbox overrides + ${substrateSandboxFields('general', g.sandboxSubstrate)} +
Clear on Kick ${settingHelpMark('agent-configuration', 'anatomy-of-an-agent', 'Clear on Kick', `Send /clear before each governor kick to reset conversation context. Prevents stale context buildup across runs. On by default for all agents.`)} @@ -66556,6 +66559,69 @@

Escalation Breaker

markDirty('security', key, n); } + const SUBSTRATE_SANDBOX_FIELDS = [ + { key: 'endpoint', label: 'Endpoint', kind: 'text', global: 'agent_sandbox.substrate.endpoint', agent: 'agents..sandbox.substrate.endpoint', help: `Substrate Control API address (host:port or an http(s):// URL). Empty uses api.ate-system.svc:443.` }, + { key: 'atespace', label: 'Atespace', kind: 'text', global: 'agent_sandbox.substrate.atespace', agent: 'agents..sandbox.substrate.atespace', help: `Substrate atespace (its tenant namespace, not a Kubernetes namespace) actors are created in. Empty uses hive.` }, + { key: 'token_env', label: 'Token env', kind: 'text', global: 'agent_sandbox.substrate.token_env', agent: 'agents..sandbox.substrate.token_env', help: `Environment variable holding a Control API bearer token. Empty reads the token file instead.` }, + { key: 'token_file', label: 'Token file', kind: 'text', global: 'agent_sandbox.substrate.token_file', agent: 'agents..sandbox.substrate.token_file', help: `Projected ServiceAccount token file hive authenticates with. Empty uses /var/run/secrets/ate/token.` }, + { key: 'ca_file', label: 'CA file', kind: 'text', global: 'agent_sandbox.substrate.ca_file', agent: 'agents..sandbox.substrate.ca_file', help: `PEM bundle the Control API serving certificate chains to. Empty uses the projected CA when present, else the system roots.` }, + { key: 'insecure_tls', label: 'Insecure TLS', kind: 'bool', global: 'agent_sandbox.substrate.insecure_tls', agent: 'agents..sandbox.substrate.insecure_tls', help: `Skip Control API and router certificate verification. Dev only.` }, + { key: 'timeout_s', label: 'Timeout (seconds)', kind: 'int', global: 'agent_sandbox.substrate.timeout_s', agent: 'agents..sandbox.substrate.timeout_s', help: `Bounds one actor from create to delete. 0 uses the sandbox timeout.` }, + { key: 'router_url', label: 'Router URL', kind: 'text', global: 'agent_sandbox.substrate.router_url', agent: 'agents..sandbox.substrate.router_url', help: `atenet-router CONNECT listener hive reaches the actor bridge through. Empty uses the in-cluster default.` }, + { key: 'router_client_bundle', label: 'Router client bundle', kind: 'text', global: 'agent_sandbox.substrate.router_client_bundle', agent: 'agents..sandbox.substrate.router_client_bundle', help: `podCertificate bundle the router authenticates hive with. Empty uses the default path.` }, + { key: 'router_ca_file', label: 'Router CA file', kind: 'text', global: 'agent_sandbox.substrate.router_ca_file', agent: 'agents..sandbox.substrate.router_ca_file', help: `CA the router serving certificate chains to. Empty uses the default path.` }, + { key: 'bridge_port', label: 'Bridge port', kind: 'int', global: 'agent_sandbox.substrate.bridge_port', agent: 'agents..sandbox.substrate.bridge_port', help: `Port the in-actor bridge listens on. 0 uses 8477.` }, + { key: 'image', label: 'Image', kind: 'text', global: 'agent_sandbox.substrate.image', agent: 'agents..sandbox.substrate.image', help: `Digest-pinned sandbox image (…@sha256:…) actors run. Substrate refuses tag-only references.` }, + { key: 'bridge_command', label: 'Bridge command', kind: 'lines', global: 'agent_sandbox.substrate.bridge_command[]', agent: 'agents..sandbox.substrate.bridge_command[]', help: `Actor entrypoint that serves the bridge. One argument per line; empty uses the crux substrate-bridge subcommand.` }, + { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, + { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, + { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, + { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, + ]; + + /* substrateSandboxFields renders the Substrate runtime block for either the + global agent_sandbox.substrate (section 'security') or one agent's + sandbox.substrate override (section 'general'). */ + function substrateSandboxFields(section, sub) { + const dirtyKey = section === 'security' ? 'agentSandboxSubstrate' : 'sandboxSubstrate'; + sub = sub || {}; + return SUBSTRATE_SANDBOX_FIELDS.map(f => { + const yaml = section === 'security' ? f.global : f.agent; + const label = 'Substrate ' + f.label.toLowerCase(); + const head = '
'; + const attrs = 'data-arg0="' + section + '" data-arg1="' + dirtyKey + '" data-arg2="' + f.key + '" data-arg3="' + f.kind + '"'; + const v = sub[f.key]; + if (f.kind === 'bool') { + return head + '
'; + } + if (f.kind === 'int') { + return head + '
'; + } + if (f.kind === 'lines' || f.kind === 'map') { + const text = f.kind === 'map' + ? Object.keys(v || {}).sort().map(k => k + '=' + v[k]).join('\n') + : (v || []).join('\n'); + return head + ''; + } + return head + ''; + }).join('\n'); + } + + function markSubstrateSandbox(section, dirtyKey, field, kind, value) { + const pending = _configState.dirty[section] && _configState.dirty[section][dirtyKey]; + const base = pending || ((_configState.data && _configState.data[section]) || {})[dirtyKey] || {}; + const next = Object.assign({}, base); + if (kind === 'bool') next[field] = !!value; + else if (kind === 'int') next[field] = Math.max(0, Math.floor(Number(value) || 0)); + else if (kind === 'lines') next[field] = String(value || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean); + else if (kind === 'map') next[field] = parseSettingsMap(value); + else next[field] = String(value || '').trim(); + if (_configState.data && _configState.data[section]) _configState.data[section][dirtyKey] = next; + markDirty(section, dirtyKey, next); + } + function parseSecurityList(value) { return String(value || '').split(',').map(s => s.trim()).filter(Boolean); } @@ -66957,6 +67023,9 @@

Escalation Breaker

+
Substrate sandbox + ${substrateSandboxFields('security', sec.agentSandboxSubstrate)} +
`; /* Backup encryption key: painted asynchronously from presence-only From 0cf675d3ac119866b6db295dc4ae6dcfcd4da6fc Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:55:11 -0700 Subject: [PATCH 030/223] =?UTF-8?q?=F0=9F=90=9B=20claude=20auth:=20share?= =?UTF-8?q?=20refreshed=20logins=20across=20per-agent=20.claude=20homes=20?= =?UTF-8?q?without=20sharing=20memory=20(#17767)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 claude auth: share refreshed logins across per-agent .claude homes without sharing memory Since the per-agent CLI homes landed, each per-UID agent has a real ~/.claude directory with only .credentials.json symlinked to the shared login at /data/home/.claude/.credentials.json. Claude Code refreshes its token by writing a temp file in ~/.claude and renaming it over .credentials.json, then chmod 0600. The rename replaces the symlink with a private, agent-owned file. bridgeHomeEntry refuses to clobber a regular file, so nothing ever re-links it; the refreshed access token and the rotated refresh grant stay in that agent's private file, the shared credential ages out, and the rest of the fleet logs out every ~8h (Discord report by Danathar, 2026-11-11). Andy's direction: keep per-agent memory isolated, make an exception for the login only. So no opt-in whole-directory sharing. Instead the hive repairs the bridge in-process: - repairClaudeCredentialBridge: when the agent's credential path is a regular file, read it (directly, or as its owner through the existing su-exec seam when the server cannot open an agent's 0600 file — the per-UID case), move it aside as .credentials.json.stale-, promote it into the shared login only if it is complete, usable and expires later than the shared token (a failed refresh's empty-token file never wins; ties keep shared), preserving the previous shared file, then restore the symlink. Legacy whole-directory ~/.claude symlinks and unreadable private files are left untouched, with a once-per-version log. - Triggered at agent launch (setupPerAgentCLIHome), every 5s from the credential watchdog (HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL) so a refresh is fleet-visible before siblings try the rotated grant, and right after a dashboard sign-in (ReloadClaudeToken). - Audited as claude_credential_rebridged (no token material). - claude.ReadTokens/ParseTokens/TokensUsable split out of LoadTokens so "cannot open" is distinguishable from "no token". Tests reproduce the CLI's tmp+rename+chmod and show the shared file did not update before the fix and does after; cover newer-shared, failed refresh, su-exec owner read, unreadable fail-safe, legacy layout, stale name collisions, watchdog cycle and sign-in trigger. Refs hivecommons/hive#17039, hivecommons/hive#17253, hivecommons/hive#17678, hivecommons/hive#17752, hivecommons/hive#17753 Signed-off-by: Andrew Anderson * 🌱 keep manager_tokens.go at its file-size ratchet baseline Move the sign-in re-bridge body into claude_credential_bridge.go and defer it from ReloadClaudeToken (LIFO: after the Unlock). Signed-off-by: Andrew Anderson * 🐛 harden shared-home permission guard and credential promotion against non-regular files Shared logins and CLI configs written by the hive (.claude/.credentials.json, .copilot/config.json, the durable Copilot token) now go through a new crux/sharedfile package: a randomly named temporary beside the target, mode and owner set and verified on the open descriptor, rename, directory fsync, and the write is refused unless the target is absent or a regular file in a regular directory the server owns. Credential promotion writes a canonical re-encoding of the parsed token block and checks the moved-aside source is the same regular file before and after. Parity with the v6 fail-safe behaviour: an unreadable private credential is left in place except on a dashboard sign-in with a usable shared login, the refusal is logged once per file version, WriteCredentials refuses a credential without an access token, and ReadTokens treats malformed JSON as nothing usable. entrypoint.sh: the shared dot-dirs get the sticky bit (not under HIVE_SHARED_AGENT_HOME=1); the single-file repairs act only on a regular file in a regular directory and WARN once otherwise; the remaining root chmod/chown sites under agent-writable directories gained the same checks. The Go permissions watcher keeps setgid/sticky when widening a directory's group bits. Shell tests drive the shipped guard functions with non-regular entries. Refs hivecommons/hive#17039, hivecommons/hive#17253, hivecommons/hive#17678, hivecommons/hive#17748 Signed-off-by: Andrew Anderson * 🌱 wait for the watchdog re-bridge with testutil.Eventually instead of a polling sleep Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../fixed-claude-shared-credentials.md | 1 + src/crux/agent/audit.go | 2 + src/crux/agent/claude_credential_bridge.go | 636 +++++++++++++ .../agent/claude_credential_bridge_test.go | 853 ++++++++++++++++++ src/crux/agent/copilottoken/copilottoken.go | 20 +- src/crux/agent/coverage_boost_test.go | 26 + src/crux/agent/credentials_watchdog.go | 35 + src/crux/agent/interactive_home.go | 6 + src/crux/agent/manager_credentials.go | 25 +- src/crux/agent/manager_tokens.go | 10 +- src/crux/agent/permissions_watcher.go | 4 +- src/crux/audit.go | 9 + src/crux/claude/oauth.go | 63 +- .../claude/oauth_write_error_branches_test.go | 62 +- src/crux/claude/usable_token_test.go | 65 ++ src/crux/sharedfile/sharedfile.go | 278 ++++++ src/crux/sharedfile/sharedfile_test.go | 480 ++++++++++ src/deploy/entrypoint.sh | 108 ++- src/deploy/test_entrypoint_perm_guard.sh | 84 ++ src/docs/agent-configuration.md | 14 + src/docs/env-vars.md | 1 + src/pkg/dashboard/audit.go | 1 + 22 files changed, 2725 insertions(+), 58 deletions(-) create mode 100644 changelog.d/fixed-claude-shared-credentials.md create mode 100644 src/crux/agent/claude_credential_bridge.go create mode 100644 src/crux/agent/claude_credential_bridge_test.go create mode 100644 src/crux/sharedfile/sharedfile.go create mode 100644 src/crux/sharedfile/sharedfile_test.go diff --git a/changelog.d/fixed-claude-shared-credentials.md b/changelog.d/fixed-claude-shared-credentials.md new file mode 100644 index 0000000000..db3fa495ed --- /dev/null +++ b/changelog.d/fixed-claude-shared-credentials.md @@ -0,0 +1 @@ +- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. diff --git a/src/crux/agent/audit.go b/src/crux/agent/audit.go index b0572db5ba..41dc7b0b09 100644 --- a/src/crux/agent/audit.go +++ b/src/crux/agent/audit.go @@ -27,6 +27,8 @@ const ( AuditCopilotTokenMissing = crux.AuditCopilotTokenMissing AuditClaudeTokenMissing = crux.AuditClaudeTokenMissing + + AuditClaudeCredentialRebridged = crux.AuditClaudeCredentialRebridged ) // auditActorSystem attributes an event to the hive process itself rather than diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go new file mode 100644 index 0000000000..b511e8d955 --- /dev/null +++ b/src/crux/agent/claude_credential_bridge.go @@ -0,0 +1,636 @@ +package agent + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "sync" + "syscall" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/sharedfile" +) + +// Claude credential bridge: share the LOGIN across per-agent .claude homes +// without sharing anything else. +// +// Since hivecommons/hive#17039 each per-UID claude agent has a REAL ~/.claude +// (its own transcripts, todos, history and auto-memory) with one symlink in +// it, ~/.claude/.credentials.json -> /data/home/.claude/.credentials.json, so +// one operator sign-in serves the fleet. Claude Code does not write through +// that link. When it refreshes the OAuth token (about every 8 hours) it writes +// a temp file in ~/.claude and renames it over .credentials.json, which +// REPLACES the symlink with a regular file owned by the agent's uid, mode +// 0600. The refresh also rotates the grant: the shared file now holds a dead +// refresh token, the fresh one is private to that agent, and every other +// agent fails its own refresh within the access token's lifetime. On per-UID +// hives the whole fleet logged out about every 8 hours. +// +// Claude Code offers no supported way to keep the link: CLAUDE_CONFIG_DIR +// moves the WHOLE config directory (settings, transcripts, auto-memory) and +// would undo the per-agent memory split; a bind mount cannot be renamed over +// (EBUSY) and needs mount privileges the pod does not have. So hive cannot +// prevent the replacement and instead DETECTS and PUBLISHES it: at every agent +// launch, on a short poll from the credential watchdog, and right after a +// dashboard sign-in. +// +// The repair never deletes a token. It moves the private file aside, keeps +// whichever of the two credentials is newer AND usable in the shared file, and +// restores the symlink: +// +// - the private copy wins only when it is complete and usable +// (claudeBridgePromotable: access and refresh tokens present, an expiresAt +// recorded, and claude.TokensUsable) and either the shared credential is +// not usable or the private access token expires LATER than the shared +// one. A refresh mints a later expiresAt, so this is what promotes one +// agent's fresh refresh to the whole fleet instead of stranding everyone +// else on the grant it just rotated. A failed refresh leaves an +// empty-token, expiresAt-0 file; it never qualifies; +// - otherwise the shared file stays as it is (ties keep the shared file, so +// a private copy identical to it is simply moved aside). +// +// Who reads the private file. The hive server runs as its own uid (dev) and +// cannot open another uid's 0600 file directly. It CAN run a helper as that +// uid: the same su-exec seam seedClaudeSessionForAgent uses to adopt an +// agent-owned .claude.json (runAsAgentUser, gated to the hive-launch group). +// So the judgement happens in-process, as the file's owner, with no root +// helper and no widening of the private file's mode. When neither read works +// (no su-exec on this deployment) the repair fails SAFE: the private file is +// left exactly where it is, the agent keeps running on the token it already +// had, and one ERROR per agent and file version says so. +// +// The private file is renamed to .credentials.json.stale- in +// the agent's own .claude directory and keeps its owner and mode; when it is +// promoted, the shared file it replaces is preserved the same way beside the +// shared credential. +// +// The repair never acts on a link path that resolves to the shared credential +// itself (an agent whose whole ~/.claude is still the legacy directory symlink +// to /data/home/.claude): that "private file" is the fleet's login. Such an +// agent is skipped with one warning; its next launch retires the directory +// link (setupPerAgentCLIHome) before the repair runs. + +// Repair triggers, recorded in the audit detail and the log. +const ( + claudeBridgeTriggerLaunch = "launch" + claudeBridgeTriggerWatchdog = "watchdog" + claudeBridgeTriggerSignIn = "sign-in" +) + +// claudeCredentialStaleInfix names a credential moved aside by the repair. +const claudeCredentialStaleInfix = ".stale-" + +// claudeSharedCredentialMode is the mode of a shared credential the repair +// writes: owner and group read/write, the same as claude.WriteCredentials (the +// dashboard sign-in) and what the entrypoint guard asserts, so every agent uid +// reaches it through the node group. +const claudeSharedCredentialMode = 0o660 + +// claudeBridgeStaleNameAttempts bounds the search for a free +// .stale-[-N] name beside a credential. +const claudeBridgeStaleNameAttempts = 100 + +// claudeBridgeMaxCredentialBytes caps a credential file the repair will read. +// A real credentials file is well under 4 KiB. +const claudeBridgeMaxCredentialBytes = 64 << 10 + +// claudeBridgeNow stamps the moved-aside file names; a seam for tests. +var claudeBridgeNow = time.Now + +// claudeBridgeMu serializes repairs: the launch path, the watchdog and the +// sign-in hook can all reach the same agent's link. It is never held while +// taking m.mu, so it cannot invert against the launch path, which already +// holds m.mu when it provisions the home. +var claudeBridgeMu sync.Mutex + +// claudeBridgeRepair describes one repaired bridge, for the audit and log. +type claudeBridgeRepair struct { + agent string + // kept is "shared" when the shared credential was kept as it was, or + // "agent" when the agent's private copy was newer and usable and was + // promoted into the shared file. + kept string + // staleAgentPath is where the private file was moved. + staleAgentPath string + // staleSharedPath is where the replaced shared file was preserved; empty + // unless kept == "agent" and a shared file existed. + staleSharedPath string +} + +// claudeCredentialBridgePaths returns the agent's bridged credential path and +// the shared target it must link to. ok is false when the agent has no +// per-agent home (no uid, an inference backend, or HIVE_SHARED_AGENT_HOME=1), +// i.e. exactly when setupInteractiveHome provisions no bridge. +func claudeCredentialBridgePaths(agentName string, uid int, backend string) (link, target string, ok bool) { + if uid <= 0 || IsInferenceBackend(backend) || sharedAgentHomeForced() { + return "", "", false + } + link = filepath.Join(interactiveHomePath(agentName), claudeHomeDirName, claudeCredentialsFileName) + return link, sharedClaudeBridgeTarget(), true +} + +// sharedClaudeBridgeTarget is the shared credential every bridge links to. +// Derived from sharedAgentHome (not the claude.CredentialsPath constant) so it +// follows the same test seam as the links themselves. +func sharedClaudeBridgeTarget() string { + return filepath.Join(sharedAgentHome, claudeHomeDirName, claudeCredentialsFileName) +} + +// errClaudeBridgeLinkIsTarget: the agent's credential path IS the shared +// credential, so there is no private copy to move aside and no link to +// restore (a legacy whole-directory ~/.claude symlink). Moving that file aside +// would strand the fleet. +var errClaudeBridgeLinkIsTarget = errors.New("credential link path resolves to the shared credential it would link to") + +// errClaudeBridgeAgentUnreadable: the agent's private credential exists but +// neither this process nor a helper running as the file's owner could read +// it. Its token cannot be judged, so the repair leaves it exactly where it is. +var errClaudeBridgeAgentUnreadable = errors.New("agent's private credential cannot be read") + +// errClaudeBridgeSourceChanged: the file renamed aside is not the regular +// file that was inspected a moment earlier. It is put back and not read. +var errClaudeBridgeSourceChanged = errors.New("private credential changed under the repair") + +// errClaudeBridgeNotPromotable: promoteClaudeCredential re-checks the bytes it +// is about to write and refuses an incomplete or unusable credential. +var errClaudeBridgeNotPromotable = errors.New("refusing to promote an incomplete or unusable credential over the shared login") + +// claudeBridgeLegacyWarned holds the credential links the legacy-layout skip +// has already been logged for, so the watchdog, which walks every managed +// agent each cycle, logs it once per agent instead of every cycle. Keyed by +// the link path, which names the agent. +var claudeBridgeLegacyWarned sync.Map + +// claudeBridgeUnreadableWarned holds, per credential link, the modification +// time of the unreadable private file last logged, so the poll logs once per +// agent and file version instead of every cycle. A new refresh (a new mtime) +// is logged again. +var claudeBridgeUnreadableWarned sync.Map + +// claudeBridgeUnsafeWarned holds the shared credential paths whose location +// sharedfile.Replace refused, so the refusal is logged once, not per cycle. +var claudeBridgeUnsafeWarned sync.Map + +// claudeBridgeLinkIsTarget reports whether link names the shared credential +// itself: the same path, or a file of the same name in a directory that +// resolves to the target's directory (a legacy whole-directory ~/.claude +// symlink). An error means a directory could not be resolved (absent, or not +// reachable by this process); callers treat that as nothing to repair. +func claudeBridgeLinkIsTarget(link, target string) (bool, error) { + if filepath.Clean(link) == filepath.Clean(target) { + return true, nil + } + if filepath.Base(link) != filepath.Base(target) { + return false, nil + } + linkDir, err := filepath.EvalSymlinks(filepath.Dir(link)) + if err != nil { + return false, err + } + targetDir, err := filepath.EvalSymlinks(filepath.Dir(target)) + if err != nil { + return false, err + } + if linkDir == targetDir { + return true, nil + } + // Different resolved paths can still be one directory (a bind mount). + li, err := os.Stat(linkDir) + if err != nil { + return false, err + } + ti, err := os.Stat(targetDir) + if err != nil { + return false, err + } + return os.SameFile(li, ti), nil +} + +// claudeBridgePromotable reports whether t may replace the shared login: a +// complete credential (access token, refresh token and a recorded expiresAt) +// that is usable now. A failed refresh leaves an empty-token, expiresAt-0 +// file, which never qualifies; neither does a token with no recorded expiry, +// which cannot be judged newer than anything. +func claudeBridgePromotable(t *claude.OAuthTokens, now time.Time) bool { + return t != nil && t.AccessToken != "" && t.RefreshToken != "" && t.ExpiresAt > 0 && + claude.TokensUsable(t, now) +} + +// preferAgentClaudeCredential decides which credential the shared file keeps. +// See the file comment for the rule. +func preferAgentClaudeCredential(agent, shared *claude.OAuthTokens, now time.Time) bool { + if !claudeBridgePromotable(agent, now) { + return false + } + if !claude.TokensUsable(shared, now) { + return true + } + return agent.ExpiresAt > shared.ExpiresAt +} + +// readClaudeCredentialAsOwner reads an agent's private credential: directly +// when this process may, otherwise through su-exec as the file's owner (the +// per-UID case: an agent-owned 0600 file and a server running as dev). +// claudeSessionOwnerSpec fails closed — a symlink, a root-owned or an +// oversized file never gets a helper — so a planted link at the credential +// path cannot make the helper read something else. +func readClaudeCredentialAsOwner(path string) ([]byte, error) { + data, err := sharedfile.ReadNoFollow(path, claudeBridgeMaxCredentialBytes) + if err == nil || !errors.Is(err, fs.ErrPermission) { + return data, err + } + spec, serr := claudeSessionOwnerSpec(path) + if serr != nil { + return nil, errors.Join(err, serr) + } + owned, oerr := runAsAgentUser(spec, nil, "cat", path) + if oerr != nil { + return nil, errors.Join(err, oerr) + } + if len(owned) > claudeBridgeMaxCredentialBytes { + return nil, fmt.Errorf("%w: %s returned %d bytes as owner", sharedfile.ErrTooLarge, path, len(owned)) + } + return owned, nil +} + +// canonicalClaudeCredential decodes data as a credentials file and re-encodes +// only its typed token block. What reaches the shared file is this encoding, +// never the bytes read from an agent's directory: a file that is not a +// credential in the shape claude.Credentials describes is not promotable. +func canonicalClaudeCredential(data []byte, now time.Time) ([]byte, *claude.OAuthTokens, error) { + if len(data) > claudeBridgeMaxCredentialBytes { + return nil, nil, errClaudeBridgeNotPromotable + } + tok := claude.ParseTokens(data) + if !claudeBridgePromotable(tok, now) { + return nil, nil, errClaudeBridgeNotPromotable + } + out, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: tok}) + if err != nil { + return nil, nil, errClaudeBridgeNotPromotable + } + return out, tok, nil +} + +// repairClaudeCredentialBridge re-bridges one agent's credential when a regular +// file has replaced the symlink. It returns nil, nil when there is nothing to +// repair (a symlink, an absent link, a directory, which is not this failure, +// or a link or target directory that cannot be resolved). It returns an error +// wrapping errClaudeBridgeLinkIsTarget, and touches nothing, when the link +// path resolves to the shared credential itself, and, unless trigger is +// claudeBridgeTriggerSignIn, one wrapping errClaudeBridgeAgentUnreadable, +// again touching nothing, when the private file cannot be read. +// +// The sign-in trigger is the exception: a dashboard sign-in has just written +// a new shared login, newer than any private copy by construction, so on that +// trigger an unreadable private file is still moved aside (unread) and the +// agent relinked to the shared login. +func (m *Manager) repairClaudeCredentialBridge(agentName, link, target, trigger string) (*claudeBridgeRepair, error) { + claudeBridgeMu.Lock() + defer claudeBridgeMu.Unlock() + + info, err := os.Lstat(link) + if err != nil || !info.Mode().IsRegular() { + return nil, nil + } + + // Never operate on the shared credential itself. Checked BEFORE anything + // is moved: when the link path resolves to the target, the "private file" + // below IS the shared login, moving it aside strands the fleet, and the + // symlink restored afterwards would point at its own path. + same, err := claudeBridgeLinkIsTarget(link, target) + if err != nil { + m.logger.Debug("claude credential bridge: cannot resolve the link or shared directory; not repairing", + "agent", agentName, "link", link, "target", target, "error", err) + return nil, nil + } + if same { + return nil, fmt.Errorf("%w: %s", errClaudeBridgeLinkIsTarget, link) + } + + // Never move aside a file whose token cannot be judged: it may be the + // only fresh token in the fleet. + if _, rerr := readClaudeCredentialAsOwner(link); rerr != nil { + if errors.Is(rerr, fs.ErrNotExist) { + // Gone since the Lstat: the CLI is mid-rename. Next cycle. + return nil, nil + } + if trigger == claudeBridgeTriggerSignIn && claude.HasUsableToken(target) { + return m.relinkUnreadableAfterSignIn(agentName, link, target) + } + return nil, fmt.Errorf("%w: %w", errClaudeBridgeAgentUnreadable, rerr) + } + now := claudeBridgeNow() + + // Move the private file aside FIRST, then judge the moved file: after the + // rename the CLI can no longer change what is being judged (its next + // write lands at link), so the decision works on a stable snapshot. + staleAgent, err := moveAsideClaudeCredential(link, now) + if err != nil { + return nil, fmt.Errorf("move private credential aside: %w", err) + } + agentData, rerr := readClaudeCredentialAsOwner(staleAgent) + if rerr != nil { + // The file renamed is not the one read above (the CLI replaced it in + // between) and this one cannot be read either. Put it back unless the + // CLI has already written yet another file there. + if _, lerr := os.Lstat(link); errors.Is(lerr, fs.ErrNotExist) { + if berr := os.Rename(staleAgent, link); berr != nil { + m.logger.Error("claude credential bridge: an unreadable private credential could not be put back; it is kept at the stale path", + "agent", agentName, "stale", staleAgent, "error", berr) + } + } + return nil, fmt.Errorf("%w: %w", errClaudeBridgeAgentUnreadable, rerr) + } + rep := &claudeBridgeRepair{agent: agentName, kept: "shared", staleAgentPath: staleAgent} + + if preferAgentClaudeCredential(claude.ParseTokens(agentData), claude.LoadTokens(target), now) { + staleShared, perr := promoteClaudeCredential(agentData, target, now) + if perr != nil { + // Put the private file back: the agent's copy was the only usable + // token, and linking the agent to the older shared one would + // strand it. The next cycle tries again. + if berr := os.Rename(staleAgent, link); berr != nil { + m.logger.Error("claude credential bridge: promotion failed and the private credential could not be restored; it is kept at the stale path", + "agent", agentName, "stale", staleAgent, "error", perr, "restore_error", berr) + } + return nil, fmt.Errorf("promote private credential to shared: %w", perr) + } + rep.kept = "agent" + rep.staleSharedPath = staleShared + } + + if err := os.Symlink(target, link); err != nil { + // EEXIST: the CLI wrote a fresh file between the rename and now. The + // next cycle repairs that one; nothing is lost either way. + return rep, fmt.Errorf("restore credential symlink: %w", err) + } + return rep, nil +} + +// claudeBridgeStaleName returns a free .stale-[-N] name +// beside path, or "" when none is free. exists reports whether a candidate is +// taken; callers that create the file atomically pass a function that tries +// the create and reports EEXIST. +func claudeBridgeStaleName(path string, now time.Time, exists func(string) (bool, error)) (string, error) { + base := path + claudeCredentialStaleInfix + now.UTC().Format("20060102T150405Z") + for i := 0; i < claudeBridgeStaleNameAttempts; i++ { + dst := base + if i > 0 { + dst = fmt.Sprintf("%s-%d", base, i) + } + taken, err := exists(dst) + if err != nil { + return "", err + } + if !taken { + return dst, nil + } + } + return "", fmt.Errorf("no free stale name for %s", path) +} + +// moveAsideClaudeCredential renames path to a unique stale name beside it and +// returns the new path. A rename keeps the file's owner and mode: the private +// copy stays readable only by the agent that wrote it. The file is identified +// before the rename and checked after it (same device and inode, still a +// regular file); anything else is put back and reported as +// errClaudeBridgeSourceChanged, unread. +func moveAsideClaudeCredential(path string, now time.Time) (string, error) { + before, err := os.Lstat(path) + if err != nil { + return "", err + } + if !before.Mode().IsRegular() { + return "", fmt.Errorf("%w: %s is %s", errClaudeBridgeSourceChanged, path, before.Mode()) + } + dst, err := claudeBridgeStaleName(path, now, func(dst string) (bool, error) { + _, lerr := os.Lstat(dst) + if lerr == nil { + return true, nil + } + if errors.Is(lerr, fs.ErrNotExist) { + return false, nil + } + return false, lerr + }) + if err != nil { + return "", err + } + if err := os.Rename(path, dst); err != nil { + return "", err + } + after, err := os.Lstat(dst) + if err != nil { + return "", err + } + if !after.Mode().IsRegular() || !os.SameFile(before, after) { + if _, lerr := os.Lstat(path); errors.Is(lerr, fs.ErrNotExist) { + _ = os.Rename(dst, path) + } + return "", fmt.Errorf("%w: %s", errClaudeBridgeSourceChanged, path) + } + return dst, nil +} + +// relinkUnreadableAfterSignIn is the sign-in exception described on +// repairClaudeCredentialBridge: the private file is moved aside without being +// read and the agent is relinked to the shared login just written. +func (m *Manager) relinkUnreadableAfterSignIn(agentName, link, target string) (*claudeBridgeRepair, error) { + staleAgent, err := moveAsideClaudeCredential(link, claudeBridgeNow()) + if err != nil { + return nil, fmt.Errorf("move private credential aside: %w", err) + } + rep := &claudeBridgeRepair{agent: agentName, kept: "shared", staleAgentPath: staleAgent} + if err := os.Symlink(target, link); err != nil { + return rep, fmt.Errorf("restore credential symlink: %w", err) + } + return rep, nil +} + +// promoteClaudeCredential replaces the shared credential with the token block +// decoded from data, atomically and at claudeSharedCredentialMode, owned like +// the file it replaces. The replaced file is preserved beside it first; its +// path is returned ("" when there was no shared file). The bytes are decoded +// and re-checked here (canonicalClaudeCredential), so a credential that +// changed after it was judged can never put an incomplete token over the +// shared login, and nothing but a typed token block is ever written. The +// write itself is sharedfile.Replace: a random O_EXCL|O_NOFOLLOW temporary, +// metadata by descriptor, and a refusal (sharedfile.ErrUnsafeLocation) when +// the shared directory or file is not what it should be. +func promoteClaudeCredential(data []byte, target string, now time.Time) (string, error) { + canonical, _, err := canonicalClaudeCredential(data, now) + if err != nil { + return "", err + } + // Refuse before the backup copy, not only at the write: the copy lands in + // the same directory, and the directory is part of what is being checked. + info, err := sharedfile.CheckLocation(target) + if err != nil { + return "", err + } + staleShared := "" + if info != nil { + staleShared, err = preserveClaudeCredential(target, now) + if err != nil { + return "", fmt.Errorf("preserve shared credential: %w", err) + } + } + if err := sharedfile.Replace(target, canonical, claudeSharedCredentialMode, sharedfile.Options{PreserveOwner: true}); err != nil { + return staleShared, err + } + return staleShared, nil +} + +// preserveClaudeCredential copies path to a unique stale name beside it, +// leaving path itself in place (agents keep reading it until the rename that +// replaces it). The copy is 0600: it is an archive, not a live credential. +// Neither the read nor the create follows a symlink. +func preserveClaudeCredential(path string, now time.Time) (string, error) { + data, err := sharedfile.ReadNoFollow(path, claudeBridgeMaxCredentialBytes) + if err != nil { + return "", err + } + var f *os.File + dst, err := claudeBridgeStaleName(path, now, func(dst string) (bool, error) { + var oerr error + f, oerr = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) + if errors.Is(oerr, fs.ErrExist) { + return true, nil + } + return false, oerr + }) + if err != nil { + return "", err + } + _, werr := f.Write(data) + cerr := f.Close() + if werr != nil || cerr != nil { + _ = os.Remove(dst) + return "", errors.Join(werr, cerr) + } + return dst, nil +} + +// rebridgeClaudeCredential runs the repair for one agent and records the +// outcome. It is the single entry point the launch path, the watchdog poll and +// the sign-in hook share, so all three log and audit identically. +func (m *Manager) rebridgeClaudeCredential(agentName, link, target, trigger string) *claudeBridgeRepair { + rep, err := m.repairClaudeCredentialBridge(agentName, link, target, trigger) + if errors.Is(err, errClaudeBridgeLinkIsTarget) { + // The legacy layout: skipped, and said once per agent rather than on + // every cycle. Not audited (nothing changed). + if _, seen := claudeBridgeLegacyWarned.LoadOrStore(link, true); !seen { + m.logger.Warn("claude credential bridge: this agent's .claude directory resolves to the shared login directory (a legacy whole-directory symlink), so its credential path IS the shared login; not re-bridging. The agent's next launch replaces the directory link with a per-agent directory", + "agent", agentName, "trigger", trigger, "dir", filepath.Dir(link), "shared", target) + } + return nil + } + // Any other outcome clears the mark, so a legacy layout that shows up + // again later is logged again. + claudeBridgeLegacyWarned.Delete(link) + if errors.Is(err, errClaudeBridgeAgentUnreadable) { + m.logClaudeBridgeUnreadable(agentName, link, trigger, err) + return nil + } + claudeBridgeUnreadableWarned.Delete(link) + if errors.Is(err, sharedfile.ErrUnsafeLocation) { + // The shared location is not what the repair expects; nothing was + // written and the agent keeps its private copy. Said once per target. + if _, seen := claudeBridgeUnsafeWarned.LoadOrStore(target, true); !seen { + m.logger.Error("claude credential bridge: the shared credential location was refused and nothing was written; the agent keeps its private copy", + "agent", agentName, "trigger", trigger, "shared", target, "error", err) + } + return nil + } + claudeBridgeUnsafeWarned.Delete(target) + if err != nil { + // Logged, not audited: a repair that keeps failing would otherwise add + // an audit row every cycle. + m.logger.Warn("claude credential bridge: repair failed", + "agent", agentName, "link", link, "trigger", trigger, "error", err) + return nil + } + if rep == nil { + return nil + } + m.logger.Warn("claude credential bridge: a private credential file had replaced the shared-login symlink; re-bridged", + "agent", agentName, + "trigger", trigger, + "kept", rep.kept, + "stale_agent_file", rep.staleAgentPath, + "stale_shared_file", rep.staleSharedPath, + "cause", "Claude Code token refresh writes a temp file and renames it over the link") + m.audit(AuditClaudeCredentialRebridged, agentName, auditFields( + "outcome", "success", + "trigger", trigger, + "kept", rep.kept, + "stale_agent_file", rep.staleAgentPath, + "stale_shared_file", rep.staleSharedPath, + )) + return rep +} + +// logClaudeBridgeUnreadable says, once per agent and private-file version, +// that the repair left an agent on a private credential it could not read. +// Not audited: nothing changed. +func (m *Manager) logClaudeBridgeUnreadable(agentName, link, trigger string, err error) { + var stamp int64 + if info, lerr := os.Lstat(link); lerr == nil { + stamp = info.ModTime().UnixNano() + } + if prev, loaded := claudeBridgeUnreadableWarned.Swap(link, stamp); loaded && prev == stamp { + return + } + m.logger.Error("claude credential bridge: this agent's private credential could not be read, directly or as its owner through su-exec, so its token cannot be judged; leaving the agent on its private copy (not moved aside, not re-linked). Other agents will not see this agent's token refresh until it is readable", + "agent", agentName, "path", link, "trigger", trigger, "error", err) +} + +// RepairClaudeCredentialBridges checks every per-agent Claude credential +// bridge and repairs each one a regular file has replaced. It returns the +// names of the agents it re-bridged, sorted. Agents are snapshotted under m.mu +// and repaired after it is released. +func (m *Manager) RepairClaudeCredentialBridges(trigger string) []string { + type candidate struct { + name string + uid int + backend string + } + m.mu.RLock() + cands := make([]candidate, 0, len(m.agents)) + for name, a := range m.agents { + if a == nil { + continue + } + cands = append(cands, candidate{name: name, uid: a.UID, backend: effectiveBackend(a)}) + } + m.mu.RUnlock() + sort.Slice(cands, func(i, j int) bool { return cands[i].name < cands[j].name }) + + var repaired []string + for _, c := range cands { + link, target, ok := claudeCredentialBridgePaths(c.name, c.uid, c.backend) + if !ok { + continue + } + if rep := m.rebridgeClaudeCredential(c.name, link, target, trigger); rep != nil { + repaired = append(repaired, c.name) + } + } + return repaired +} + +// rebridgeClaudeCredentialsAfterSignIn runs from ReloadClaudeToken: a sign-in +// wrote the shared file, and agents whose link a refresh replaced would never +// see it. Relink them now rather than on the next poll. A logout leaves +// nothing usable, so nothing is moved. +func (m *Manager) rebridgeClaudeCredentialsAfterSignIn() { + if claude.HasUsableToken(sharedClaudeBridgeTarget()) { + m.RepairClaudeCredentialBridges(claudeBridgeTriggerSignIn) + } +} diff --git a/src/crux/agent/claude_credential_bridge_test.go b/src/crux/agent/claude_credential_bridge_test.go new file mode 100644 index 0000000000..67928f5207 --- /dev/null +++ b/src/crux/agent/claude_credential_bridge_test.go @@ -0,0 +1,853 @@ +package agent + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" + "github.com/hivecommons/hive/crux/sharedfile" +) + +// Fake tokens. Every token string carries this sentinel so a test can prove +// no log line or audit field ever leaks token material. +const bridgeTestTokenSentinel = "SENTINEL-TOKEN-MATERIAL" + +func bridgeTestCreds(t *testing.T, tag string, expiresAt int64) []byte { + t.Helper() + data, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: &claude.OAuthTokens{ + AccessToken: "acc-" + tag + "-" + bridgeTestTokenSentinel, + RefreshToken: "ref-" + tag + "-" + bridgeTestTokenSentinel, + ExpiresAt: expiresAt, + }}) + if err != nil { + t.Fatal(err) + } + return data +} + +// bridgeTestFixture provisions a per-agent home for "scanner" with the shared +// login in place and the credential symlink bridged, and returns the manager, +// the audit sink, the link path and the shared path. +func bridgeTestFixture(t *testing.T) (*Manager, *fakeAuditSink, *AgentProcess, string, string) { + t.Helper() + shared := withSharedAgentHome(t) + m, sink := testManagerWithSink(t) + ap := &AgentProcess{Name: "scanner", UID: 1001, Config: config.AgentConfig{Backend: "claude"}} + m.mu.Lock() + m.agents["scanner"] = ap + m.mu.Unlock() + target := filepath.Join(shared, ".claude", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + m.setupInteractiveHome(ap, "claude") + link := filepath.Join(interactiveHomePath("scanner"), ".claude", ".credentials.json") + if got, err := os.Readlink(link); err != nil || got != target { + t.Fatalf("fixture: credential bridge not provisioned: target=%q err=%v", got, err) + } + return m, sink, ap, link, target +} + +// simulateClaudeRefresh does what Claude Code's credential store does on a +// token refresh: write a temp file in the agent's .claude and rename it over +// .credentials.json, then chmod 0600. The rename replaces the symlink. +func simulateClaudeRefresh(t *testing.T, link string, data []byte) { + t.Helper() + tmp := link + ".tmp.12345" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmp, link); err != nil { + t.Fatal(err) + } + if err := os.Chmod(link, 0o600); err != nil { + t.Fatal(err) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("simulated refresh did not replace the link: info=%v err=%v", info, err) + } +} + +func readTokensAt(t *testing.T, path string) *claude.OAuthTokens { + t.Helper() + tok := claude.LoadTokens(path) + if tok == nil { + t.Fatalf("no tokens at %s", path) + } + return tok +} + +func staleFilesBeside(t *testing.T, path string) []string { + t.Helper() + matches, err := filepath.Glob(path + claudeCredentialStaleInfix + "*") + if err != nil { + t.Fatal(err) + } + return matches +} + +func assertLinkRestored(t *testing.T, link, target string) { + t.Helper() + got, err := os.Readlink(link) + if err != nil || got != target { + t.Fatalf("credential link not restored: target=%q err=%v", got, err) + } +} + +// TestSetupInteractiveHome_RefreshReplacedLinkIsPublishedToSharedLogin is the +// reported failure (Discord, 2026-11-11): on a per-UID hive one agent's token +// refresh replaced its credential link, the refresh never reached the shared +// file, and the fleet logged out when the shared token expired. The refreshed +// token must reach the shared login and the agent must be re-linked, with +// memory untouched. +func TestSetupInteractiveHome_RefreshReplacedLinkIsPublishedToSharedLogin(t *testing.T) { + m, sink, ap, link, target := bridgeTestFixture(t) + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "old", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + memory := filepath.Join(filepath.Dir(link), "projects", "-data-agents-scanner", "memory", "MEMORY.md") + if err := os.MkdirAll(filepath.Dir(memory), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(memory, []byte("# mine"), 0o644); err != nil { + t.Fatal(err) + } + + fresh := bridgeTestCreds(t, "fresh", now.Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, fresh) + // The defect: the shared file did not move. + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-old-") { + t.Fatalf("precondition: shared file should still hold the old token, got %q", tok.AccessToken) + } + + m.setupInteractiveHome(ap, "claude") + + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-fresh-") { + t.Errorf("refresh not published to the shared login: shared access token = %q", tok.AccessToken) + } + assertLinkRestored(t, link, target) + info, err := os.Stat(target) + if err != nil || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("shared credential mode = %v err=%v, want %#o", info.Mode(), err, claudeSharedCredentialMode) + } + // Private copy moved aside, not deleted, still owner-only. + stale := staleFilesBeside(t, link) + if len(stale) != 1 { + t.Fatalf("stale agent files = %v, want exactly one", stale) + } + if info, err := os.Stat(stale[0]); err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("stale private copy mode = %v err=%v, want 0600", info.Mode(), err) + } + // Old shared file preserved beside the shared credential. + if got := staleFilesBeside(t, target); len(got) != 1 || !bytes.Contains(mustRead(t, got[0]), []byte("acc-old-")) { + t.Errorf("replaced shared file not preserved: %v", got) + } + if got, err := os.ReadFile(memory); err != nil || string(got) != "# mine" { + t.Errorf("per-agent memory disturbed: %q err=%v", got, err) + } + // Audited, kept=agent, no token material. + rec := sink.find(AuditClaudeCredentialRebridged) + if rec == nil { + t.Fatal("no claude_credential_rebridged audit entry") + } + if rec.Agent != "scanner" || rec.Fields["kept"] != "agent" || rec.Fields["trigger"] != claudeBridgeTriggerLaunch { + t.Errorf("audit entry = %+v", rec) + } + if strings.Contains(fmt.Sprint(rec.Fields), bridgeTestTokenSentinel) { + t.Error("audit fields carry token material") + } + // Idempotent: a second launch finds a healthy link and changes nothing. + m.setupInteractiveHome(ap, "claude") + if sink.count() != 1 { + t.Errorf("second launch audited again: %d entries", sink.count()) + } +} + +func mustRead(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return data +} + +// TestRepairClaudeCredentialBridge_KeepsNewerShared: a dashboard sign-in after +// the refresh wrote a newer shared credential; the agent's stale private copy +// is moved aside and the agent re-linked, the shared file untouched. +func TestRepairClaudeCredentialBridge_KeepsNewerShared(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + now := time.Now() + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "agent", now.Add(1*time.Hour).UnixMilli())) + sharedBytes := bridgeTestCreds(t, "signin", now.Add(8*time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + + rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep == nil || rep.kept != "shared" || rep.staleSharedPath != "" { + t.Fatalf("repair = %+v, want kept=shared with no preserved shared file", rep) + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential was rewritten") + } + assertLinkRestored(t, link, target) + if got := staleFilesBeside(t, link); len(got) != 1 || !bytes.Contains(mustRead(t, got[0]), []byte("acc-agent-")) { + t.Errorf("private copy not moved aside: %v", got) + } + if rec := sink.find(AuditClaudeCredentialRebridged); rec == nil || rec.Fields["kept"] != "shared" { + t.Errorf("audit = %+v", rec) + } +} + +// TestRepairClaudeCredentialBridge_FailedRefreshNeverWins: an agent whose +// refresh failed on a rotated grant writes an empty-token, expiresAt-0 file. +// It must never replace the shared login, usable or not. +func TestRepairClaudeCredentialBridge_FailedRefreshNeverWins(t *testing.T) { + for _, sharedUsable := range []bool{true, false} { + t.Run(fmt.Sprintf("sharedUsable=%v", sharedUsable), func(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + exp := time.Now().Add(-time.Hour).UnixMilli() + if sharedUsable { + exp = time.Now().Add(time.Hour).UnixMilli() + } + sharedBytes := bridgeTestCreds(t, "shared", exp) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, []byte(`{"claudeAiOauth":{"accessToken":"","refreshToken":"","expiresAt":0}}`)) + + rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep == nil || rep.kept != "shared" { + t.Fatalf("repair = %+v, want kept=shared", rep) + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("empty-token file replaced the shared login") + } + assertLinkRestored(t, link, target) + }) + } +} + +// TestRepairClaudeCredentialBridge_ReadsAgentOwnedFileAsOwner is the per-UID +// case: the private file is the agent's, 0600, and the hive process cannot +// open it. The read must go through the su-exec seam as the file's owner, and +// the fresh token must still be promoted. +func TestRepairClaudeCredentialBridge_ReadsAgentOwnedFileAsOwner(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can read a mode-000 file; the permission fallback cannot be exercised") + } + m, _, _, link, target := bridgeTestFixture(t) + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "old", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + fresh := bridgeTestCreds(t, "fresh", now.Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, fresh) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + + var mu sync.Mutex + var calls []string + old := runAsAgentUser + runAsAgentUser = func(spec string, stdin []byte, argv ...string) ([]byte, error) { + mu.Lock() + defer mu.Unlock() + calls = append(calls, spec+" "+strings.Join(argv, " ")) + if len(argv) != 2 || argv[0] != "cat" { + return nil, fmt.Errorf("unexpected argv %v", argv) + } + // Stand in for the owner: the file is ours, only its mode blocks us. + if err := os.Chmod(argv[1], 0o600); err != nil { + return nil, err + } + defer func() { _ = os.Chmod(argv[1], 0) }() + return os.ReadFile(argv[1]) + } + t.Cleanup(func() { runAsAgentUser = old }) + + rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep == nil || rep.kept != "agent" { + t.Fatalf("repair = %+v, want kept=agent", rep) + } + if !bytes.Equal(mustRead(t, target), fresh) { + t.Error("agent-owned refresh was not promoted to the shared login") + } + assertLinkRestored(t, link, target) + mu.Lock() + defer mu.Unlock() + wantSpec := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) + if len(calls) < 1 || !strings.HasPrefix(calls[0], wantSpec+" cat "+link) { + t.Errorf("su-exec calls = %v, want first read as %s of %s", calls, wantSpec, link) + } +} + +// TestRepairClaudeCredentialBridge_UnreadablePrivateLeftInPlace: when neither +// a direct read nor the owner helper can read the private file, nothing is +// moved or relinked (the agent keeps the token it has), nothing is audited, +// and the ERROR is logged once per file version. +func TestRepairClaudeCredentialBridge_UnreadablePrivateLeftInPlace(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can read a mode-000 file; the permission fallback cannot be exercised") + } + m, sink, _, link, target := bridgeTestFixture(t) + var logBuf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + private := bridgeTestCreds(t, "private", time.Now().Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, private) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(link, 0o600) }) + old := runAsAgentUser + runAsAgentUser = func(string, []byte, ...string) ([]byte, error) { + return nil, errors.New("su-exec unavailable") + } + t.Cleanup(func() { runAsAgentUser = old }) + + for i := 0; i < 3; i++ { + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("cycle %d: repair = %+v, want none", i, rep) + } + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Errorf("private file was moved or relinked: info=%v err=%v", info, err) + } + if got := staleFilesBeside(t, link); len(got) != 0 { + t.Errorf("stale files created: %v", got) + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential changed") + } + if sink.count() != 0 { + t.Errorf("audited %d entries, want none (nothing changed)", sink.count()) + } + logs := logBuf.String() + if n := strings.Count(logs, "cannot be judged"); n != 1 { + t.Errorf("unreadable ERROR logged %d times over three cycles, want once:\n%s", n, logs) + } + if strings.Contains(logs, bridgeTestTokenSentinel) { + t.Error("log carries token material") + } + if !strings.Contains(logs, link) { + t.Error("log does not name the path") + } + // A new file version is logged again. + if err := os.Chmod(link, 0o600); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, private) + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(link, future, future); err != nil { + t.Fatal(err) + } + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if n := strings.Count(logBuf.String(), "cannot be judged"); n != 2 { + t.Errorf("new file version not logged again: %d", n) + } +} + +// TestRepairClaudeCredentialBridge_LegacyLayoutIsNeverTouched: an agent whose +// whole ~/.claude is still the legacy directory symlink has the SHARED login +// at its credential path. Moving it aside would log the fleet out. +func TestRepairClaudeCredentialBridge_LegacyLayoutIsNeverTouched(t *testing.T) { + shared := withSharedAgentHome(t) + m, sink := testManagerWithSink(t) + var logBuf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + target := filepath.Join(shared, ".claude", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + home := interactiveHomePath("scanner") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(shared, ".claude"), filepath.Join(home, ".claude")); err != nil { + t.Fatal(err) + } + link := filepath.Join(home, ".claude", ".credentials.json") + + for i := 0; i < 2; i++ { + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("repair = %+v, want none", rep) + } + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential disturbed") + } + if got := staleFilesBeside(t, target); len(got) != 0 { + t.Errorf("shared login moved aside: %v", got) + } + if sink.count() != 0 { + t.Error("legacy skip must not audit") + } + if n := strings.Count(logBuf.String(), "legacy whole-directory symlink"); n != 1 { + t.Errorf("legacy warning logged %d times, want once", n) + } +} + +// TestRepairClaudeCredentialBridge_NothingToRepair: a healthy link, an absent +// link and a directory are not this failure. +func TestRepairClaudeCredentialBridge_NothingToRepair(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Errorf("healthy link: repair = %+v", rep) + } + if err := os.Remove(link); err != nil { + t.Fatal(err) + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Errorf("absent link: repair = %+v", rep) + } + if err := os.Mkdir(link, 0o755); err != nil { + t.Fatal(err) + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Errorf("directory: repair = %+v", rep) + } + if sink.count() != 0 { + t.Error("nothing to repair must not audit") + } + // No per-agent home at all: no paths, no repair. + if _, _, ok := claudeCredentialBridgePaths("scanner", 0, "claude"); ok { + t.Error("uid 0 must have no bridge") + } + if _, _, ok := claudeCredentialBridgePaths("scanner", 1001, "vllm"); ok { + t.Error("inference backend must have no bridge") + } + t.Setenv("HIVE_SHARED_AGENT_HOME", "1") + if _, _, ok := claudeCredentialBridgePaths("scanner", 1001, "claude"); ok { + t.Error("forced shared home must have no bridge") + } +} + +// TestRepairClaudeCredentialBridge_StaleNamesNeverCollide: two repairs within +// the same second get distinct stale names on both sides. +func TestRepairClaudeCredentialBridge_StaleNamesNeverCollide(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + fixed := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) + oldNow := claudeBridgeNow + claudeBridgeNow = func() time.Time { return fixed } + t.Cleanup(func() { claudeBridgeNow = oldNow }) + if err := os.WriteFile(target, bridgeTestCreds(t, "s0", fixed.Add(time.Hour).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + for i := 1; i <= 2; i++ { + simulateClaudeRefresh(t, link, bridgeTestCreds(t, fmt.Sprintf("a%d", i), fixed.Add(time.Duration(i+1)*time.Hour).UnixMilli())) + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep == nil || rep.kept != "agent" { + t.Fatalf("repair %d = %+v", i, rep) + } + } + if got := staleFilesBeside(t, link); len(got) != 2 { + t.Errorf("agent stale files = %v, want 2 distinct", got) + } + got := staleFilesBeside(t, target) + if len(got) != 2 { + t.Fatalf("shared stale files = %v, want 2 distinct", got) + } + if !strings.HasSuffix(got[1], "-1") { + t.Errorf("second stale name = %q, want -1 suffix", got[1]) + } + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-a2-") { + t.Errorf("shared token = %q, want the latest refresh", tok.AccessToken) + } +} + +func TestPreferAgentClaudeCredential(t *testing.T) { + now := time.Now() + live := func(tag string, d time.Duration) *claude.OAuthTokens { + return &claude.OAuthTokens{AccessToken: "a" + tag, RefreshToken: "r" + tag, ExpiresAt: now.Add(d).UnixMilli()} + } + cases := []struct { + name string + agent, shared *claude.OAuthTokens + want bool + }{ + {"nil agent", nil, live("s", time.Hour), false}, + {"agent newer", live("a", 2*time.Hour), live("s", time.Hour), true}, + {"agent older", live("a", time.Hour), live("s", 2*time.Hour), false}, + {"tie keeps shared", live("a", time.Hour), live("a", time.Hour), false}, + {"shared unusable", live("a", time.Hour), &claude.OAuthTokens{AccessToken: "x", ExpiresAt: now.Add(-time.Hour).UnixMilli()}, true}, + {"shared absent", live("a", time.Hour), nil, true}, + {"agent no refresh token", &claude.OAuthTokens{AccessToken: "a", ExpiresAt: now.Add(time.Hour).UnixMilli()}, nil, false}, + {"agent no expiry", &claude.OAuthTokens{AccessToken: "a", RefreshToken: "r"}, nil, false}, + {"agent expired, no refresh expiry", &claude.OAuthTokens{AccessToken: "a", RefreshToken: "r", ExpiresAt: now.Add(-time.Hour).UnixMilli()}, nil, true}, + {"agent empty", &claude.OAuthTokens{}, nil, false}, + } + for _, tc := range cases { + if got := preferAgentClaudeCredential(tc.agent, tc.shared, now); got != tc.want { + t.Errorf("%s: prefer agent = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestPromoteClaudeCredential_RefusesUnusableBytes(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, ".credentials.json") + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + for _, bad := range [][]byte{[]byte("not json"), []byte(`{}`), []byte(`{"claudeAiOauth":{"accessToken":"","refreshToken":"","expiresAt":0}}`)} { + stale, err := promoteClaudeCredential(bad, target, time.Now()) + if !errors.Is(err, errClaudeBridgeNotPromotable) || stale != "" { + t.Errorf("promote(%q) = (%q, %v), want errClaudeBridgeNotPromotable", bad, stale, err) + } + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential changed") + } + if got := staleFilesBeside(t, target); len(got) != 0 { + t.Errorf("preserved copies made for refused promotions: %v", got) + } + // Promotion with no shared file: nothing to preserve, chown seam called + // only when an owner was read. + fresh := filepath.Join(dir, "fresh", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(fresh), 0o755); err != nil { + t.Fatal(err) + } + stale, err := promoteClaudeCredential(sharedBytes, fresh, time.Now()) + if err != nil || stale != "" { + t.Fatalf("promote to absent target = (%q, %v)", stale, err) + } + if info, err := os.Stat(fresh); err != nil || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("promoted mode = %v err=%v", info.Mode(), err) + } +} + +func TestPromoteClaudeCredential_WritesOnlyTheTokenBlock(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, ".credentials.json") + if err := os.WriteFile(target, bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + exp := time.Now().Add(8 * time.Hour).UnixMilli() + // A credential with extra material around and inside the token block. + raw := fmt.Sprintf(`{"claudeAiOauth":{"accessToken":"acc-%s","refreshToken":"ref-%s","expiresAt":%d,"scopes":["user:inference"],"extraField":{"deep":"EXTRA-MATERIAL"}},"otherTopLevel":"EXTRA-MATERIAL"}`, + bridgeTestTokenSentinel, bridgeTestTokenSentinel, exp) + stale, err := promoteClaudeCredential([]byte(raw), target, time.Now()) + if err != nil || stale == "" { + t.Fatalf("promote = (%q, %v)", stale, err) + } + got := mustRead(t, target) + if bytes.Contains(got, []byte("EXTRA-MATERIAL")) { + t.Errorf("shared file carries bytes outside the typed token block: %s", got) + } + want, _ := json.Marshal(claude.Credentials{ClaudeAIOAuth: &claude.OAuthTokens{ + AccessToken: "acc-" + bridgeTestTokenSentinel, RefreshToken: "ref-" + bridgeTestTokenSentinel, + ExpiresAt: exp, Scopes: []string{"user:inference"}, + }}) + if !bytes.Equal(got, want) { + t.Errorf("shared file = %s, want canonical %s", got, want) + } + if info, err := os.Lstat(target); err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("shared file mode = %v err=%v", info.Mode(), err) + } + if info, err := os.Stat(stale); err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("preserved shared copy mode = %v err=%v, want 0600", info.Mode(), err) + } + if _, err := os.Lstat(target + ".tmp-rebridge"); !errors.Is(err, fs.ErrNotExist) { + t.Error("a predictable temporary name was used") + } + if left, _ := filepath.Glob(target + ".*.tmp"); len(left) != 0 { + t.Errorf("temporary files left: %v", left) + } + if _, _, err := canonicalClaudeCredential(bytes.Repeat([]byte("x"), claudeBridgeMaxCredentialBytes+1), time.Now()); !errors.Is(err, errClaudeBridgeNotPromotable) { + t.Errorf("oversized input: err = %v", err) + } +} + +// TestRepairClaudeCredentialBridge_SharedTargetSymlinkIsRefused: the shared +// credential path holds a symlink. Nothing is written through it, the agent +// keeps its private copy in place, and the refusal is logged once. +func TestRepairClaudeCredentialBridge_SharedTargetSymlinkIsRefused(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + var logBuf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, target); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "fresh", time.Now().Add(8*time.Hour).UnixMilli())) + + for i := 0; i < 2; i++ { + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("repair = %+v, want refusal", rep) + } + } + if got, _ := os.ReadFile(victim); string(got) != "keep" { + t.Errorf("victim rewritten: %q", got) + } + if info, _ := os.Lstat(victim); info.Mode().Perm() != 0o600 { + t.Errorf("victim mode changed: %v", info.Mode()) + } + if info, err := os.Lstat(target); err != nil || info.Mode()&fs.ModeSymlink == 0 { + t.Errorf("symlink at the shared path was replaced: %v %v", info, err) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Errorf("agent's private file not left in place: %v %v", info, err) + } + if got := staleFilesBeside(t, link); len(got) != 0 { + t.Errorf("private copy left aside: %v", got) + } + if sink.count() != 0 { + t.Error("a refusal must not audit") + } + if n := strings.Count(logBuf.String(), "location was refused"); n != 1 { + t.Errorf("refusal logged %d times over two cycles, want once:\n%s", n, logBuf.String()) + } + if strings.Contains(logBuf.String(), bridgeTestTokenSentinel) { + t.Error("log carries token material") + } +} + +// TestRepairClaudeCredentialBridge_SharedDirSymlinkIsRefused: the shared +// .claude directory itself is a symlink. Nothing is written into what it +// points at. +func TestRepairClaudeCredentialBridge_SharedDirSymlinkIsRefused(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + dir := filepath.Dir(target) + elsewhere := filepath.Join(t.TempDir(), "elsewhere") + if err := os.Rename(dir, elsewhere); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, dir); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "fresh", time.Now().Add(8*time.Hour).UnixMilli())) + + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("repair = %+v, want refusal", rep) + } + if !bytes.Equal(mustRead(t, filepath.Join(elsewhere, ".credentials.json")), sharedBytes) { + t.Error("wrote through the directory symlink") + } + entries, _ := os.ReadDir(elsewhere) + for _, e := range entries { + if e.Name() != ".credentials.json" { + t.Errorf("unexpected entry behind the directory symlink: %s", e.Name()) + } + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Errorf("agent's private file not left in place: %v %v", info, err) + } +} + +// TestMoveAsideClaudeCredential_NonRegularSourceIsRefused: a symlink where the +// private file should be is neither moved nor read. +func TestMoveAsideClaudeCredential_NonRegularSourceIsRefused(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim") + if err := os.WriteFile(victim, bridgeTestCreds(t, "v", time.Now().Add(time.Hour).UnixMilli()), 0o600); err != nil { + t.Fatal(err) + } + src := filepath.Join(dir, ".credentials.json") + if err := os.Symlink(victim, src); err != nil { + t.Fatal(err) + } + if _, err := moveAsideClaudeCredential(src, time.Now()); !errors.Is(err, errClaudeBridgeSourceChanged) { + t.Errorf("err = %v, want errClaudeBridgeSourceChanged", err) + } + if info, err := os.Lstat(src); err != nil || info.Mode()&fs.ModeSymlink == 0 { + t.Errorf("symlink moved: %v %v", info, err) + } + if got := staleFilesBeside(t, src); len(got) != 0 { + t.Errorf("stale files created: %v", got) + } + // A directory is refused the same way. + d := filepath.Join(dir, "d") + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + if _, err := moveAsideClaudeCredential(d, time.Now()); !errors.Is(err, errClaudeBridgeSourceChanged) { + t.Errorf("directory: err = %v", err) + } + // The owner-read path refuses a symlink too, even one to a readable file. + if _, err := readClaudeCredentialAsOwner(src); !errors.Is(err, sharedfile.ErrNotRegular) { + t.Errorf("read through symlink: err = %v, want ErrNotRegular", err) + } +} + +// TestRepairClaudeCredentialBridge_UnreadableIsRelinkedOnlyAfterSignIn: an +// unreadable private file stays put on the watchdog, but a dashboard sign-in +// (a shared login newer than any private copy) moves it aside unread and +// relinks the agent. +func TestRepairClaudeCredentialBridge_UnreadableIsRelinkedOnlyAfterSignIn(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can read a mode-000 file; the permission fallback cannot be exercised") + } + m, sink, _, link, target := bridgeTestFixture(t) + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "private", time.Now().Add(8*time.Hour).UnixMilli())) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + old := runAsAgentUser + runAsAgentUser = func(string, []byte, ...string) ([]byte, error) { return nil, errors.New("no su-exec") } + t.Cleanup(func() { runAsAgentUser = old }) + + // No shared login yet: even the sign-in trigger moves nothing. + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerSignIn); rep != nil { + t.Fatalf("sign-in without a shared login: repair = %+v", rep) + } + sharedBytes := bridgeTestCreds(t, "signin", time.Now().Add(8*time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("watchdog: repair = %+v, want none", rep) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("watchdog moved the unreadable file: %v %v", info, err) + } + + m.ReloadClaudeToken() + assertLinkRestored(t, link, target) + stale := staleFilesBeside(t, link) + if len(stale) != 1 { + t.Fatalf("stale files = %v, want the unreadable copy moved aside", stale) + } + if info, err := os.Lstat(stale[0]); err != nil || info.Mode().Perm() != 0 { + t.Errorf("moved-aside copy mode = %v err=%v, want unchanged (000)", info.Mode(), err) + } + _ = os.Chmod(stale[0], 0o600) + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared login changed") + } + rec := sink.find(AuditClaudeCredentialRebridged) + if rec == nil || rec.Fields["kept"] != "shared" || rec.Fields["trigger"] != claudeBridgeTriggerSignIn { + t.Errorf("audit = %+v", rec) + } +} + +// TestCredentialWatchdog_RebridgesWithinOneCycle: the bridge poll repairs a +// replaced link without waiting for the 5-minute probe. +func TestCredentialWatchdog_RebridgesWithinOneCycle(t *testing.T) { + t.Setenv(ClaudeBridgeIntervalEnv, "20ms") + t.Setenv(CredentialWatchdogIntervalEnv, "1h") + m, _, _, link, target := bridgeTestFixture(t) + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "old", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + fresh := bridgeTestCreds(t, "fresh", now.Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, fresh) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { m.StartCredentialWatchdog(ctx); close(done) }() + + testutil.Eventually(t, 5*time.Second, func() bool { + got, err := os.Readlink(link) + return err == nil && got == target + }, "watchdog did not re-bridge the credential within 5s") + cancel() + <-done + if !bytes.Equal(mustRead(t, target), fresh) { + t.Error("watchdog did not publish the refresh to the shared login") + } +} + +func TestClaudeBridgeInterval(t *testing.T) { + t.Setenv(ClaudeBridgeIntervalEnv, "") + if got := claudeBridgeInterval(); got != defaultClaudeBridgeInterval { + t.Errorf("default = %v", got) + } + t.Setenv(ClaudeBridgeIntervalEnv, "250ms") + if got := claudeBridgeInterval(); got != 250*time.Millisecond { + t.Errorf("override = %v", got) + } + for _, bad := range []string{"nope", "-1s", "0"} { + t.Setenv(ClaudeBridgeIntervalEnv, bad) + if got := claudeBridgeInterval(); got != defaultClaudeBridgeInterval { + t.Errorf("%q: = %v, want default", bad, got) + } + } +} + +// TestReloadClaudeToken_RebridgesAfterSignIn: a dashboard sign-in reaches an +// agent whose link a refresh had replaced, at once rather than on the next +// poll; a logout (nothing usable) moves nothing. +func TestReloadClaudeToken_RebridgesAfterSignIn(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "agent", time.Now().Add(-time.Hour).UnixMilli())) + + // Logout: no shared file. Nothing moves. + m.ReloadClaudeToken() + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("logout must not touch the private file: info=%v err=%v", info, err) + } + if sink.count() != 0 { + t.Fatal("logout audited a rebridge") + } + + sharedBytes := bridgeTestCreds(t, "signin", time.Now().Add(8*time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + m.ReloadClaudeToken() + assertLinkRestored(t, link, target) + rec := sink.find(AuditClaudeCredentialRebridged) + if rec == nil || rec.Fields["trigger"] != claudeBridgeTriggerSignIn || rec.Fields["kept"] != "shared" { + t.Errorf("audit = %+v", rec) + } +} + +// TestRepairClaudeCredentialBridge_MidRenameIsNextCycle: the link vanishes +// between the Lstat and the read (the CLI is mid-rename). Nothing to do. +func TestRepairClaudeCredentialBridge_MidRenameIsNextCycle(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "agent", time.Now().Add(time.Hour).UnixMilli())) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + old := runAsAgentUser + runAsAgentUser = func(spec string, stdin []byte, argv ...string) ([]byte, error) { + _ = os.Remove(link) + return nil, &fs.PathError{Op: "open", Path: link, Err: fs.ErrNotExist} + } + t.Cleanup(func() { runAsAgentUser = old }) + if os.Getuid() == 0 { + // Root reads the file directly; remove it from under the Lstat another way. + runAsAgentUser = old + t.Skip("root cannot exercise the owner-read path") + } + rep, err := m.repairClaudeCredentialBridge("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep != nil || err != nil { + t.Errorf("mid-rename: (%+v, %v), want nothing", rep, err) + } +} diff --git a/src/crux/agent/copilottoken/copilottoken.go b/src/crux/agent/copilottoken/copilottoken.go index 480036a3d3..de66bcbfb7 100644 --- a/src/crux/agent/copilottoken/copilottoken.go +++ b/src/crux/agent/copilottoken/copilottoken.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/sharedfile" ) // ConfigHasTokens returns true if either the Copilot config or Claude @@ -134,19 +135,18 @@ func ReadConfig(path string) (map[string]interface{}, error) { } // WriteConfig marshals cfg back to config.json with the CLI's // header -// preserved and the CLI-expected mode. Written via a temp file + rename so a -// concurrent CLI read never sees a half-written file. +// preserved and the CLI-expected mode. Written through sharedfile.Replace (a +// random temporary beside path, renamed into place) so a concurrent CLI read +// never sees a half-written file, and so a non-regular file at path, or a +// directory that is not this process's own, is refused rather than written +// through: config.json lives in a directory every agent uid can write to. func WriteConfig(path string, cfg map[string]interface{}, mode os.FileMode) error { out, err := json.MarshalIndent(cfg, "", " ") if err != nil { return err } content := ConfigHeader + string(out) - tmp := path + ".tmp" - if err := os.WriteFile(tmp, []byte(content), mode); err != nil { - return err - } - return os.Rename(tmp, path) + return sharedfile.Replace(path, []byte(content), mode, sharedfile.Options{PreserveOwner: true}) } // ClearExpiredTokens removes stored copilot tokens from config.json. @@ -361,9 +361,5 @@ func WriteDurableToken(durablePath, token string) error { if token == "" { return nil } - tmp := durablePath + ".tmp" - if err := os.WriteFile(tmp, []byte(token), 0o600); err != nil { - return err - } - return os.Rename(tmp, durablePath) + return sharedfile.Replace(durablePath, []byte(token), 0o600, sharedfile.Options{}) } diff --git a/src/crux/agent/coverage_boost_test.go b/src/crux/agent/coverage_boost_test.go index ec674b8f83..e5ee3839ed 100644 --- a/src/crux/agent/coverage_boost_test.go +++ b/src/crux/agent/coverage_boost_test.go @@ -1844,6 +1844,32 @@ func TestClearExpiredTokens_MissingFile(t *testing.T) { // fixSharedConfigPerms — with actual path // --------------------------------------------------------------------------- +// A symlink at the shared config path is left alone: the file it points at +// keeps its mode. +func TestFixSharedConfigPerms_SymlinkIsNotFollowed(t *testing.T) { + cleanup := configTestHelper(t) + defer cleanup() + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, sharedCopilotConfigPath); err != nil { + t.Fatal(err) + } + m := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "claude"}, + }, discardLogger(), ProjectContext{}) + m.mu.RLock() + agent := m.agents["scanner"] + m.mu.RUnlock() + + m.fixSharedConfigPerms(agent) + + if info, _ := os.Stat(victim); info.Mode().Perm() != 0o600 { + t.Fatalf("victim mode = %v, want 0600 untouched", info.Mode()) + } +} + func TestFixSharedConfigPerms_FixesPerms(t *testing.T) { cleanup := configTestHelper(t) defer cleanup() diff --git a/src/crux/agent/credentials_watchdog.go b/src/crux/agent/credentials_watchdog.go index f64c76eca0..5a3c02d85e 100644 --- a/src/crux/agent/credentials_watchdog.go +++ b/src/crux/agent/credentials_watchdog.go @@ -25,6 +25,19 @@ const ( // only for overrides) — disabling is intentionally not offered so the // safety net cannot be silently turned off. CredentialWatchdogIntervalEnv = "HIVE_CREDENTIAL_WATCHDOG_INTERVAL" + + // defaultClaudeBridgeInterval is how often the watchdog checks each + // per-agent Claude credential link for a token refresh that replaced it + // (claude_credential_bridge.go). Short on purpose: every agent holds the + // SAME access token, so they all reach expiry together and refresh within + // a minute or two of each other; the first refresh rotates the grant, and + // each agent that tries before that refresh is published fails and signs + // itself out. The check is one Lstat per claude agent, so 5s costs + // nothing; the su-exec read happens only when a link has been replaced. + defaultClaudeBridgeInterval = 5 * time.Second + // ClaudeBridgeIntervalEnv overrides the bridge poll interval with a Go + // duration string. + ClaudeBridgeIntervalEnv = "HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL" ) // credentialWatchdogInterval resolves the watchdog interval from @@ -38,6 +51,17 @@ func credentialWatchdogInterval() time.Duration { return defaultCredentialWatchdogInterval } +// claudeBridgeInterval resolves the bridge poll interval from +// ClaudeBridgeIntervalEnv, falling back to the default. +func claudeBridgeInterval() time.Duration { + if v := os.Getenv(ClaudeBridgeIntervalEnv); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultClaudeBridgeInterval +} + // credentialWatch describes one CLI backend whose usable credential lives in a // durable file on the hive PVC that the hive relies on but does NOT itself keep // alive. Only backends of this shape belong here: bob resolves its key at @@ -199,6 +223,8 @@ func credentialWatches() []credentialWatch { func (m *Manager) StartCredentialWatchdog(ctx context.Context) { ticker := time.NewTicker(credentialWatchdogInterval()) defer ticker.Stop() + bridgeTicker := time.NewTicker(claudeBridgeInterval()) + defer bridgeTicker.Stop() // Per-backend transition tracker so we log/audit on TRANSITIONS (usable -> // unusable and back) rather than every tick — a standing condition would // otherwise flood the Audit Log at the tick rate. @@ -207,7 +233,16 @@ func (m *Manager) StartCredentialWatchdog(ctx context.Context) { select { case <-ctx.Done(): return + case <-bridgeTicker.C: + if m.backendInUse("claude") { + m.RepairClaudeCredentialBridges(claudeBridgeTriggerWatchdog) + } case <-ticker.C: + // Re-bridge before probing, so a just-published refresh is what + // the shared-credential probe sees. + if m.backendInUse("claude") { + m.RepairClaudeCredentialBridges(claudeBridgeTriggerWatchdog) + } for _, w := range credentialWatches() { m.evalCredentialWatch(w, lastUnusable) } diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index 114e404d1e..4b9798400d 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -425,6 +425,12 @@ func (m *Manager) setupPerAgentCLIHome(agentName, home string, uid int, spec per for _, name := range spec.bridges { link := filepath.Join(dir, name) target := filepath.Join(shared, name) + if spec.dir == claudeHomeDirName && name == claudeCredentialsFileName { + // A token refresh since the last launch replaced the link with a + // private file; publish it to the shared login and relink BEFORE + // bridgeHomeEntry, which refuses to clobber a regular file. + m.rebridgeClaudeCredential(agentName, link, target, claudeBridgeTriggerLaunch) + } if _, err := os.Stat(target); err != nil && isPerAgentCLIBridgeDir(spec.dir, name) { // Absent target: a dangling FILE link goes live on first login; a // dangling DIRECTORY link would make the CLI's own mkdir -p fail, diff --git a/src/crux/agent/manager_credentials.go b/src/crux/agent/manager_credentials.go index b681e0a9c2..232ba58d45 100644 --- a/src/crux/agent/manager_credentials.go +++ b/src/crux/agent/manager_credentials.go @@ -2,14 +2,16 @@ package agent import ( "context" + "errors" "fmt" - "os" + "io/fs" "strings" "time" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/claude" "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/sharedfile" ) // sharedCopilotConfigPath and sharedClaudeCredentialPath are vars (not consts) @@ -304,19 +306,22 @@ func compactPaneTail(output string, maxLines int) string { // fixSharedConfigPerms ensures /data/home/.copilot/config.json is group-readable // before launching an agent. Copilot CLI rewrites this file with 600 perms on // token refresh, locking out other agent UIDs that share the same HOME. +// +// The chmod goes through an open descriptor (sharedfile.Chmod), so anything +// other than a regular file at that path is left alone and logged. func (m *Manager) fixSharedConfigPerms(agent *AgentProcess) { - info, err := os.Stat(sharedCopilotConfigPath) - if err != nil { + prev, changed, err := sharedfile.Chmod(sharedCopilotConfigPath, sharedConfigDesiredMode) + switch { + case errors.Is(err, fs.ErrNotExist): return - } - if info.Mode().Perm() == sharedConfigDesiredMode { + case err != nil: + m.logger.Warn("failed to fix config.json perms", "agent", agent.Name, "error", err) + return + case !changed: return } - m.logger.Warn("fixing shared config.json perms before launch", + m.logger.Warn("fixed shared config.json perms before launch", "agent", agent.Name, - "was", fmt.Sprintf("%04o", info.Mode().Perm()), + "was", fmt.Sprintf("%04o", prev), "fix", fmt.Sprintf("%04o", sharedConfigDesiredMode)) - if err := os.Chmod(sharedCopilotConfigPath, sharedConfigDesiredMode); err != nil { - m.logger.Warn("failed to fix config.json perms", "error", err) - } } diff --git a/src/crux/agent/manager_tokens.go b/src/crux/agent/manager_tokens.go index d6d5a23444..44cc4933dc 100644 --- a/src/crux/agent/manager_tokens.go +++ b/src/crux/agent/manager_tokens.go @@ -21,18 +21,18 @@ type ( AgentMintIssuer = crux.AgentMintIssuer ) -// ReloadClaudeToken re-reads the Claude credentials file and updates the -// cached token. Called by the dashboard after a successful OAuth login. +// ReloadClaudeToken re-reads the Claude credentials file, updates the cached +// token and (after m.mu is released) re-bridges agents a refresh unlinked. func (m *Manager) ReloadClaudeToken() { + defer m.rebridgeClaudeCredentialsAfterSignIn() // LIFO: runs after Unlock m.mu.Lock() defer m.mu.Unlock() m.claudeAuthToken = claude.ReadAccessToken(claude.CredentialsPath) } // Copilot token sources, as reported by CopilotTokenSource. Operator-facing -// phrases: they are spliced verbatim into the model-picker notice and the -// "rejected by upstream" log line, so each one has to read as an answer to -// "which credential did GitHub just reject?". +// phrases, spliced verbatim into the model-picker notice and the "rejected by +// upstream" log line: each must answer "which credential did GitHub reject?". const ( // Defined in the crux root package; see crux.CopilotTokenSourceEnv. CopilotTokenSourceEnv = crux.CopilotTokenSourceEnv diff --git a/src/crux/agent/permissions_watcher.go b/src/crux/agent/permissions_watcher.go index d8b5fbb394..648b60fb3d 100644 --- a/src/crux/agent/permissions_watcher.go +++ b/src/crux/agent/permissions_watcher.go @@ -620,7 +620,9 @@ func fixEntry(path string, fi os.FileInfo, logger *slog.Logger) { if fi.IsDir() { // Directories need u+rwx to be usable. if mode.Perm()&DirPerms != DirPerms { - newMode := mode.Perm() | DirPerms + // Keep setgid and sticky: the entrypoint sets both on the shared + // dot-dirs, and a repair of the group bits must not drop them. + newMode := mode&(os.ModePerm|os.ModeSetgid|os.ModeSticky) | DirPerms if err := chmod(newMode); err != nil { warnDeduped(logger, "chmod dir", "permissions watcher: chmod dir failed", path, err) } else { diff --git a/src/crux/audit.go b/src/crux/audit.go index 350cfa09e4..367c026240 100644 --- a/src/crux/audit.go +++ b/src/crux/audit.go @@ -40,6 +40,15 @@ const ( // failure shape and same operator-login recovery as the Copilot case; the // detail's outcome= distinguishes "missing" from "invalid or expired". AuditClaudeTokenMissing = "claude_token_missing" + + // AuditClaudeCredentialRebridged is recorded when an agent's per-agent + // ~/.claude/.credentials.json symlink to the shared Claude login had been + // replaced by a regular file (Claude Code saves a token refresh by renaming + // over the path) and hive re-bridged it: either promoting the agent's + // fresher token into the shared login or re-linking the agent to the + // shared one. The detail's kept= says which; it never carries token + // material. + AuditClaudeCredentialRebridged = "claude_credential_rebridged" ) // AuditSink receives agent lifecycle events for durable, queryable recording. diff --git a/src/crux/claude/oauth.go b/src/crux/claude/oauth.go index 47ef515a51..6777c081a8 100644 --- a/src/crux/claude/oauth.go +++ b/src/crux/claude/oauth.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -13,6 +14,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/hivecommons/hive/crux/sharedfile" ) const ( @@ -187,8 +190,27 @@ func ExchangeCode(code, codeVerifier, redirectURI string) (*OAuthTokens, error) }, nil } -// WriteCredentials atomically writes tokens to the credentials file. +// ErrNoAccessToken is returned by WriteCredentials for a token set with no +// access token. Writing one would replace whatever login is on file with a +// signed-out credential. +var ErrNoAccessToken = errors.New("refusing to write a Claude credential with no access token") + +// CredentialsFileMode is the mode of the shared credentials file: owner and +// group read/write, so every agent uid reaches it through the shared group. +const CredentialsFileMode = 0o660 + +// WriteCredentials atomically writes tokens to the credentials file. It +// refuses a nil token set or one with an empty access token +// (ErrNoAccessToken), so no hive caller can replace the shared login with a +// signed-out credential. The write is sharedfile.Replace: the credentials +// directory is shared with every agent uid, so the temporary name is random, +// metadata is set by descriptor, and a non-regular file at the destination or +// a directory that is not this process's own is refused rather than written +// through. func WriteCredentials(tokens *OAuthTokens, path string) error { + if tokens == nil || tokens.AccessToken == "" { + return ErrNoAccessToken + } dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o2775); err != nil { return fmt.Errorf("create credentials dir: %w", err) @@ -199,13 +221,8 @@ func WriteCredentials(tokens *OAuthTokens, path string) error { if err != nil { return fmt.Errorf("marshal credentials: %w", err) } - - tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, data, 0o660); err != nil { - return fmt.Errorf("write temp credentials: %w", err) - } - if err := os.Rename(tmpPath, path); err != nil { - return fmt.Errorf("rename credentials: %w", err) + if err := sharedfile.Replace(path, data, CredentialsFileMode, sharedfile.Options{PreserveOwner: true}); err != nil { + return fmt.Errorf("write credentials: %w", err) } return nil } @@ -214,10 +231,29 @@ func WriteCredentials(tokens *OAuthTokens, path string) error { // nil when the file is absent, malformed, or carries no token block. It never // applies an expiry rule — callers decide what a stale token means to them. func LoadTokens(path string) *OAuthTokens { - data, err := os.ReadFile(path) + tokens, err := ReadTokens(path) if err != nil { return nil } + return tokens +} + +// ReadTokens is LoadTokens with the read error kept, for callers that must +// tell "this process cannot read the file" (fs.ErrNotExist, or +// fs.ErrPermission on another uid's 0600 credential) apart from "the file +// holds no usable token". A file that was read but is malformed, or carries +// no token block, returns (nil, nil). The error never carries file content. +func ReadTokens(path string) (*OAuthTokens, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParseTokens(data), nil +} + +// ParseTokens decodes a credentials file already in memory. Nil for malformed +// bytes or a file with no token block, like LoadTokens. +func ParseTokens(data []byte) *OAuthTokens { var creds Credentials if err := json.Unmarshal(data, &creds); err != nil { return nil @@ -281,11 +317,16 @@ func HasValidToken(path string) bool { // Positive evidence only: an unreadable or absent file returns false, leaving // callers exactly where they were before they asked. func HasUsableToken(path string) bool { - tokens := LoadTokens(path) + return TokensUsable(LoadTokens(path), time.Now()) +} + +// TokensUsable is the HasUsableToken rule applied to an already-loaded token +// set at a given instant: a live access token, or a refresh grant that has not +// expired. A nil token set is never usable. +func TokensUsable(tokens *OAuthTokens, now time.Time) bool { if tokens == nil { return false } - now := time.Now() if tokens.AccessToken != "" && !expired(tokens.ExpiresAt, now) { return true } diff --git a/src/crux/claude/oauth_write_error_branches_test.go b/src/crux/claude/oauth_write_error_branches_test.go index b8b9e8881b..97a1e9a2dc 100644 --- a/src/crux/claude/oauth_write_error_branches_test.go +++ b/src/crux/claude/oauth_write_error_branches_test.go @@ -1,10 +1,14 @@ package claude import ( + "errors" "os" "path/filepath" "strings" "testing" + "time" + + "github.com/hivecommons/hive/crux/sharedfile" ) // tokenEndpoint must fall back to the real Claude TokenURL when no test @@ -35,8 +39,11 @@ func TestWriteCredentials_TempWriteError(t *testing.T) { if err == nil { t.Fatalf("WriteCredentials into read-only dir: err = nil, want error") } - if !strings.Contains(err.Error(), "write temp credentials") { - t.Fatalf("error = %q, want it to mention write temp credentials", err) + if !strings.Contains(err.Error(), "write credentials") { + t.Fatalf("error = %q, want it to mention write credentials", err) + } + if left, _ := filepath.Glob(filepath.Join(dir, "creds.json.*")); len(left) != 0 { + t.Fatalf("temporary files left behind: %v", left) } } @@ -51,9 +58,54 @@ func TestWriteCredentials_RenameError(t *testing.T) { err := WriteCredentials(&OAuthTokens{AccessToken: "at"}, path) if err == nil { - t.Fatalf("WriteCredentials over non-empty dir: err = nil, want error") + t.Fatalf("WriteCredentials over a directory: err = nil, want error") + } + if !errors.Is(err, sharedfile.ErrUnsafeLocation) { + t.Fatalf("error = %q, want sharedfile.ErrUnsafeLocation", err) + } + if left, _ := filepath.Glob(path + ".*"); len(left) != 0 { + t.Fatalf("temporary files left behind: %v", left) + } +} + +// A symlink at the credentials path is refused, and the file it points at is +// left exactly as it was. +func TestWriteCredentials_SymlinkAtPathIsRefused(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, ".credentials.json") + if err := os.Symlink(victim, path); err != nil { + t.Fatal(err) + } + err := WriteCredentials(&OAuthTokens{AccessToken: "at"}, path) + if !errors.Is(err, sharedfile.ErrUnsafeLocation) { + t.Fatalf("error = %v, want sharedfile.ErrUnsafeLocation", err) + } + if got, _ := os.ReadFile(victim); string(got) != "keep" { + t.Fatalf("victim rewritten: %q", got) + } + if info, _ := os.Lstat(victim); info.Mode().Perm() != 0o600 { + t.Fatalf("victim mode changed: %v", info.Mode()) + } +} + +// TestWriteCredentialsRefusesSignedOutTokens: no hive writer may put a +// credential with no access token over the shared login. +func TestWriteCredentialsRefusesSignedOutTokens(t *testing.T) { + path := filepath.Join(t.TempDir(), ".credentials.json") + if err := WriteCredentials(&OAuthTokens{AccessToken: "live", ExpiresAt: time.Now().Add(time.Hour).UnixMilli()}, path); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(path) + for _, tok := range []*OAuthTokens{nil, {}, {RefreshToken: "r", ExpiresAt: 1}} { + if err := WriteCredentials(tok, path); !errors.Is(err, ErrNoAccessToken) { + t.Errorf("WriteCredentials(%+v) = %v, want ErrNoAccessToken", tok, err) + } } - if !strings.Contains(err.Error(), "rename credentials") { - t.Fatalf("error = %q, want it to mention rename credentials", err) + if after, _ := os.ReadFile(path); string(after) != string(before) { + t.Error("the shared login was replaced by a signed-out credential") } } diff --git a/src/crux/claude/usable_token_test.go b/src/crux/claude/usable_token_test.go index bde265b4cc..720c474a5b 100644 --- a/src/crux/claude/usable_token_test.go +++ b/src/crux/claude/usable_token_test.go @@ -2,6 +2,8 @@ package claude import ( "encoding/json" + "errors" + "io/fs" "os" "path/filepath" "testing" @@ -132,3 +134,66 @@ func TestRefreshTokenExpiresAtRoundTrips(t *testing.T) { t.Fatalf("refreshTokenExpiresAt lost on round trip: got %+v", got) } } + +// TestReadTokens_KeepsTheReadError: the credential bridge must tell "this +// process may not open another uid's 0600 file" apart from "no token here"; +// LoadTokens flattens both to nil, ReadTokens does not. +func TestReadTokens_KeepsTheReadError(t *testing.T) { + if _, err := ReadTokens(filepath.Join(t.TempDir(), "missing")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("missing file: err = %v, want ErrNotExist", err) + } + bad := filepath.Join(t.TempDir(), ".credentials.json") + if err := os.WriteFile(bad, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if tok, err := ReadTokens(bad); err != nil || tok != nil { + t.Errorf("malformed: (%v, %v), want nil, nil (readable, nothing usable)", tok, err) + } + if tok := LoadTokens(bad); tok != nil { + t.Errorf("LoadTokens malformed = %v, want nil", tok) + } + empty := filepath.Join(t.TempDir(), ".credentials.json") + if err := os.WriteFile(empty, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if tok, err := ReadTokens(empty); err != nil || tok != nil { + t.Errorf("no token block: (%v, %v), want nil, nil", tok, err) + } + if os.Getuid() != 0 { + locked := writeCredFile(t, &OAuthTokens{AccessToken: "a", ExpiresAt: ms(time.Hour)}) + if err := os.Chmod(locked, 0); err != nil { + t.Fatal(err) + } + if _, err := ReadTokens(locked); !errors.Is(err, fs.ErrPermission) { + t.Errorf("unreadable: err = %v, want ErrPermission", err) + } + } +} + +func TestParseTokens(t *testing.T) { + if tok := ParseTokens([]byte("nope")); tok != nil { + t.Errorf("malformed = %v", tok) + } + if tok := ParseTokens([]byte(`{}`)); tok != nil { + t.Errorf("no block = %v", tok) + } + tok := ParseTokens([]byte(`{"claudeAiOauth":{"accessToken":"a","refreshToken":"r","expiresAt":42}}`)) + if tok == nil || tok.AccessToken != "a" || tok.RefreshToken != "r" || tok.ExpiresAt != 42 { + t.Errorf("parsed = %+v", tok) + } +} + +func TestTokensUsable_AtAnInstant(t *testing.T) { + now := time.Now() + later := now.Add(2 * time.Hour) + tok := &OAuthTokens{AccessToken: "a", ExpiresAt: now.Add(time.Hour).UnixMilli()} + if !TokensUsable(tok, now) { + t.Error("live access token at now should be usable") + } + if TokensUsable(tok, later) { + t.Error("the same token judged after its expiry should be unusable") + } + if TokensUsable(nil, now) { + t.Error("nil is never usable") + } +} diff --git a/src/crux/sharedfile/sharedfile.go b/src/crux/sharedfile/sharedfile.go new file mode 100644 index 0000000000..9d4fd05ce0 --- /dev/null +++ b/src/crux/sharedfile/sharedfile.go @@ -0,0 +1,278 @@ +// Package sharedfile replaces and reads files that live in directories other +// uids can write to: the shared CLI credential and config files under +// /data/home that the hive maintains for a fleet of per-UID agents. +// +// A plain "write .tmp, chmod, rename" in such a directory acts on +// whatever is at those names when each call runs. Here every step acts on the +// inode this process created, by descriptor, and the name of that inode is +// unpredictable; the directory and the destination are checked to be what the +// caller expects (a directory this process owns, a regular file or nothing) +// right before the rename, and a symlink at any of those places is a refusal, +// never something to write through. +package sharedfile + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" +) + +// ErrUnsafeLocation is wrapped by Replace when the destination directory or +// the destination itself is not in a state this package will write through: +// the directory is missing, a symlink, world-writable, or not owned by this +// process, or the destination exists and is not a regular file. Nothing is +// written when it is returned. Callers that poll should log it once, not per +// cycle. +var ErrUnsafeLocation = errors.New("refusing to replace a file at an unsafe location") + +// ErrTooLarge is returned by ReadNoFollow for a file over the caller's cap. +var ErrTooLarge = errors.New("file is larger than the read cap") + +// ErrNotRegular is returned by ReadNoFollow when the path is a symlink or any +// other non-regular file. +var ErrNotRegular = errors.New("not a regular file") + +// createAttempts bounds the search for a free temporary name. A collision on +// a 64-bit random suffix is not expected; the retries make an EEXIST from a +// deliberately pre-created name a failure rather than a write-through. +const createAttempts = 8 + +// chownFile applies an owner to the inode behind an open descriptor. A seam: +// tests run as one uid and cannot change an owner for real. +var chownFile = func(f *os.File, uid, gid int) error { return f.Chown(uid, gid) } + +// randRead fills the temporary-name suffix; a seam so tests can force a +// collision or a failure. +var randRead = rand.Read + +// syncFile flushes the written temporary; a seam so tests can fail it. +var syncFile = func(f *os.File) error { return f.Sync() } + +// Options tune Replace. +type Options struct { + // PreserveOwner re-applies the previous destination's owner to the new + // file, best-effort: an unprivileged process cannot chown, and the file it + // wrote is already readable to the group by mode. + PreserveOwner bool +} + +// Replace atomically replaces target with data at mode. The data is written +// to a new file with a random name beside target (O_EXCL|O_NOFOLLOW, 0600), +// chmodded, optionally chowned, fsynced and verified by descriptor, and then +// renamed over target; the directory is fsynced afterwards. On any failure the +// temporary file is removed and target is untouched. +// +// Before the rename, the directory must be a real directory (not a symlink) +// owned by this process and not world-writable, and target must be absent or +// a regular file (not a symlink); otherwise ErrUnsafeLocation. +func Replace(target string, data []byte, mode fs.FileMode, opts Options) error { + dir := filepath.Dir(target) + prev, err := checkLocation(dir, target) + if err != nil { + return err + } + f, tmp, err := createTemp(target) + if err != nil { + return err + } + err = fill(f, data, mode, prev, opts) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + _ = os.Remove(tmp) + return err + } + // The location can have changed while the file was being written. + if _, err := checkLocation(dir, target); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, target); err != nil { + _ = os.Remove(tmp) + return err + } + syncDir(dir) + return nil +} + +// CheckLocation verifies target's directory and target as documented on +// Replace, and returns the current target's Lstat (nil when absent). Callers +// that do preparatory work beside target (a backup copy, say) use it to refuse +// before that work, not only at the final write. +func CheckLocation(target string) (os.FileInfo, error) { + return checkLocation(filepath.Dir(target), target) +} + +// checkLocation verifies dir and target as documented on Replace and returns +// the current target's stat (nil when absent). +func checkLocation(dir, target string) (os.FileInfo, error) { + dinfo, err := os.Lstat(dir) + if err != nil { + return nil, fmt.Errorf("%w: %s: %w", ErrUnsafeLocation, dir, err) + } + if !dinfo.IsDir() { + return nil, fmt.Errorf("%w: %s is not a directory (mode %s)", ErrUnsafeLocation, dir, dinfo.Mode()) + } + if dinfo.Mode().Perm()&0o002 != 0 { + return nil, fmt.Errorf("%w: %s is world-writable (mode %s)", ErrUnsafeLocation, dir, dinfo.Mode()) + } + if st, ok := dinfo.Sys().(*syscall.Stat_t); ok { + if euid := os.Geteuid(); euid != 0 && int(st.Uid) != euid { + return nil, fmt.Errorf("%w: %s is owned by uid %d, this process is uid %d", ErrUnsafeLocation, dir, st.Uid, euid) + } + } + tinfo, err := os.Lstat(target) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("%w: %s: %w", ErrUnsafeLocation, target, err) + } + if !tinfo.Mode().IsRegular() { + return nil, fmt.Errorf("%w: %s is not a regular file (mode %s)", ErrUnsafeLocation, target, tinfo.Mode()) + } + return tinfo, nil +} + +// createTemp creates a new 0600 file with a random name beside target. The +// name is ".<16 hex>.tmp", so sweeps that match ".*.tmp" still +// recognise a leftover. +func createTemp(target string) (*os.File, string, error) { + for i := 0; i < createAttempts; i++ { + var b [8]byte + if _, err := randRead(b[:]); err != nil { + return nil, "", err + } + tmp := target + "." + hex.EncodeToString(b[:]) + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) + if errors.Is(err, fs.ErrExist) { + continue + } + if err != nil { + return nil, "", err + } + return f, tmp, nil + } + return nil, "", fmt.Errorf("no free temporary name beside %s after %d attempts", target, createAttempts) +} + +// fill writes data to f and sets its metadata, all through the descriptor, +// then verifies by fstat that the inode is a regular file with exactly the +// mode (and, when a chown succeeded, the owner) asked for. +func fill(f *os.File, data []byte, mode fs.FileMode, prev os.FileInfo, opts Options) error { + if _, err := f.Write(data); err != nil { + return err + } + if err := f.Chmod(mode); err != nil { + return err + } + wantUID, wantGID := -1, -1 + if opts.PreserveOwner && prev != nil { + if st, ok := prev.Sys().(*syscall.Stat_t); ok { + if err := chownFile(f, int(st.Uid), int(st.Gid)); err == nil { + wantUID, wantGID = int(st.Uid), int(st.Gid) + } + } + } + if err := syncFile(f); err != nil { + return err + } + info, err := f.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode().Perm() != mode.Perm() { + return fmt.Errorf("%s: written file is mode %s, want regular %s", f.Name(), info.Mode(), mode.Perm()) + } + if st, ok := info.Sys().(*syscall.Stat_t); ok { + if st.Nlink != 1 { + return fmt.Errorf("%s: written file has %d links, want 1", f.Name(), st.Nlink) + } + if wantUID >= 0 && (int(st.Uid) != wantUID || int(st.Gid) != wantGID) { + return fmt.Errorf("%s: written file is owned by %d:%d, want %d:%d", f.Name(), st.Uid, st.Gid, wantUID, wantGID) + } + } + return nil +} + +// syncDir makes the rename durable; best-effort, like every writer before it. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} + +// openRegularNoFollow opens path without following a symlink and confirms the +// opened file is regular. ErrNotRegular for a symlink or any other kind. +func openRegularNoFollow(path string, flag int) (*os.File, os.FileInfo, error) { + f, err := os.OpenFile(path, flag|syscall.O_NOFOLLOW, 0) + if err != nil { + var pe *fs.PathError + if errors.As(err, &pe) && (errors.Is(pe.Err, syscall.ELOOP) || errors.Is(pe.Err, syscall.EMLINK)) { + return nil, nil, fmt.Errorf("%w: %s is a symlink", ErrNotRegular, path) + } + return nil, nil, err + } + info, err := f.Stat() + if err == nil && !info.Mode().IsRegular() { + err = fmt.Errorf("%w: %s (mode %s)", ErrNotRegular, path, info.Mode()) + } + if err != nil { + f.Close() + return nil, nil, err + } + return f, info, nil +} + +// Chmod sets mode on the regular file at path through an open descriptor, so +// a symlink planted at path is refused (ErrNotRegular) instead of followed. +// Returns the mode the file had before, and changed=false when it already +// matched (nothing is done in that case). +func Chmod(path string, mode fs.FileMode) (prev fs.FileMode, changed bool, err error) { + f, info, err := openRegularNoFollow(path, os.O_RDONLY) + if err != nil { + return 0, false, err + } + defer f.Close() + prev = info.Mode().Perm() + if prev == mode.Perm() { + return prev, false, nil + } + if err := f.Chmod(mode.Perm()); err != nil { + return prev, false, err + } + return prev, true, nil +} + +// ReadNoFollow reads a regular file of at most max bytes without following a +// symlink at path. ErrNotRegular for a symlink or other non-regular file, +// ErrTooLarge over the cap; the open error (fs.ErrNotExist, fs.ErrPermission) +// otherwise. +func ReadNoFollow(path string, max int64) ([]byte, error) { + f, info, err := openRegularNoFollow(path, os.O_RDONLY) + if err != nil { + return nil, err + } + defer f.Close() + if info.Size() > max { + return nil, fmt.Errorf("%w: %s is %d bytes, cap %d", ErrTooLarge, path, info.Size(), max) + } + data, err := io.ReadAll(io.LimitReader(f, max+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > max { + return nil, fmt.Errorf("%w: %s grew past %d bytes while being read", ErrTooLarge, path, max) + } + return data, nil +} diff --git a/src/crux/sharedfile/sharedfile_test.go b/src/crux/sharedfile/sharedfile_test.go new file mode 100644 index 0000000000..d8307e37bc --- /dev/null +++ b/src/crux/sharedfile/sharedfile_test.go @@ -0,0 +1,480 @@ +package sharedfile + +import ( + "bytes" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func leftovers(t *testing.T, target string) []string { + t.Helper() + m, err := filepath.Glob(target + ".*.tmp") + if err != nil { + t.Fatal(err) + } + return m +} + +func modeOf(t *testing.T, path string) fs.FileMode { + t.Helper() + info, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + return info.Mode() +} + +func TestReplace_WritesAtModeAndLeavesNoTemp(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "config.json") + if err := Replace(target, []byte("one"), 0o660, Options{}); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(target); string(got) != "one" { + t.Errorf("content = %q", got) + } + if m := modeOf(t, target); !m.IsRegular() || m.Perm() != 0o660 { + t.Errorf("mode = %v, want regular 0660", m) + } + // Replacing an existing regular file works and keeps the new mode. + if err := Replace(target, []byte("two"), 0o640, Options{PreserveOwner: true}); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(target); string(got) != "two" { + t.Errorf("content after replace = %q", got) + } + if m := modeOf(t, target); m.Perm() != 0o640 { + t.Errorf("mode after replace = %v", m) + } + if l := leftovers(t, target); len(l) != 0 { + t.Errorf("temporary files left: %v", l) + } +} + +// Two writes never share a temporary name, and the name is not derivable +// from the target alone. +func TestReplace_TempNameIsUnpredictable(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + var names []string + old := chownFile + chownFile = func(f *os.File, uid, gid int) error { + names = append(names, filepath.Base(f.Name())) + return nil + } + t.Cleanup(func() { chownFile = old }) + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := Replace(target, []byte("x"), 0o600, Options{PreserveOwner: true}); err != nil { + t.Fatal(err) + } + } + if len(names) != 2 || names[0] == names[1] { + t.Fatalf("temp names = %v, want two distinct", names) + } + for _, n := range names { + if !strings.HasPrefix(n, "f.") || !strings.HasSuffix(n, ".tmp") || len(n) != len("f.")+16+len(".tmp") { + t.Errorf("temp name %q is not .<16 hex>.tmp", n) + } + } + // The sentinel a fixed-name writer would use is never created. + if _, err := os.Lstat(target + ".tmp"); !errors.Is(err, fs.ErrNotExist) { + t.Error("a predictable .tmp exists") + } +} + +func TestReplace_RefusesSymlinkAtTarget(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + target := filepath.Join(dir, "cred") + if err := os.Symlink(victim, target); err != nil { + t.Fatal(err) + } + err := Replace(target, []byte("evil"), 0o660, Options{PreserveOwner: true}) + if !errors.Is(err, ErrUnsafeLocation) { + t.Fatalf("err = %v, want ErrUnsafeLocation", err) + } + if got, _ := os.ReadFile(victim); string(got) != "keep" { + t.Errorf("victim rewritten: %q", got) + } + if m := modeOf(t, victim); m.Perm() != 0o600 { + t.Errorf("victim mode changed: %v", m) + } + if m := modeOf(t, target); m&fs.ModeSymlink == 0 { + t.Error("symlink at target was replaced") + } + if l := leftovers(t, target); len(l) != 0 { + t.Errorf("temporary files left: %v", l) + } +} + +func TestReplace_RefusesUnsafeDirectory(t *testing.T) { + real := t.TempDir() + // A symlink where the directory should be. + linkDir := filepath.Join(t.TempDir(), "home") + if err := os.Symlink(real, linkDir); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(linkDir, "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("symlinked dir: err = %v, want ErrUnsafeLocation", err) + } + if entries, _ := os.ReadDir(real); len(entries) != 0 { + t.Errorf("wrote through the directory symlink: %v", entries) + } + // A missing directory. + if err := Replace(filepath.Join(real, "nope", "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("missing dir: err = %v", err) + } + // A world-writable directory. + ww := filepath.Join(t.TempDir(), "ww") + if err := os.Mkdir(ww, 0o777); err != nil { + t.Fatal(err) + } + if err := os.Chmod(ww, 0o1777); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(ww, "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("world-writable dir: err = %v", err) + } + // A file where the directory should be. + plain := filepath.Join(t.TempDir(), "plain") + if err := os.WriteFile(plain, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(plain, "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("file as dir: err = %v", err) + } + // A directory where the target should be. + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "d"), 0o700); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(dir, "d"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("dir as target: err = %v", err) + } +} + +func TestReplace_ChownFailureIsBestEffortAndVerified(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + old := chownFile + chownFile = func(*os.File, int, int) error { return errors.New("EPERM") } + t.Cleanup(func() { chownFile = old }) + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); err != nil { + t.Fatalf("a failed chown must not fail the write: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "new" { + t.Errorf("content = %q", got) + } + // A chown that "succeeds" but leaves the wrong owner is caught by fstat. + chownFile = func(*os.File, int, int) error { return nil } + victimOwner := filepath.Join(dir, "g") + if err := os.WriteFile(victimOwner, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + // Pretend the previous file belonged to someone else by faking prev. + prev := fakeInfo{FileInfo: must(os.Lstat(victimOwner)), uid: 12345, gid: 12345} + f, tmp, err := createTemp(victimOwner) + if err != nil { + t.Fatal(err) + } + err = fill(f, []byte("x"), 0o660, prev, Options{PreserveOwner: true}) + _ = f.Close() + _ = os.Remove(tmp) + if err == nil || !strings.Contains(err.Error(), "owned by") { + t.Errorf("owner mismatch not detected: %v", err) + } +} + +type fakeInfo struct { + os.FileInfo + uid, gid uint32 +} + +func (f fakeInfo) Sys() any { + st, _ := f.FileInfo.Sys().(*syscall.Stat_t) + cp := *st + cp.Uid, cp.Gid = f.uid, f.gid + return &cp +} + +func must(info os.FileInfo, err error) os.FileInfo { + if err != nil { + panic(err) + } + return info +} + +func TestCreateTemp_SkipsTakenNames(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + f, tmp, err := createTemp(target) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + if m := modeOf(t, tmp); m.Perm() != 0o600 { + t.Errorf("temp mode = %v, want 0600", m) + } + // A second create beside the first gets another name. + g, tmp2, err := createTemp(target) + if err != nil { + t.Fatal(err) + } + _ = g.Close() + if tmp == tmp2 { + t.Error("same temp name twice") + } +} + +func TestReadNoFollow(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "f") + if err := os.WriteFile(p, []byte("hello"), 0o600); err != nil { + t.Fatal(err) + } + got, err := ReadNoFollow(p, 64) + if err != nil || !bytes.Equal(got, []byte("hello")) { + t.Errorf("read = %q, %v", got, err) + } + if _, err := ReadNoFollow(p, 4); !errors.Is(err, ErrTooLarge) { + t.Errorf("over cap: err = %v", err) + } + link := filepath.Join(dir, "l") + if err := os.Symlink(p, link); err != nil { + t.Fatal(err) + } + if _, err := ReadNoFollow(link, 64); !errors.Is(err, ErrNotRegular) { + t.Errorf("symlink: err = %v, want ErrNotRegular", err) + } + if _, err := ReadNoFollow(dir, 64); !errors.Is(err, ErrNotRegular) { + t.Errorf("directory: err = %v, want ErrNotRegular", err) + } + if _, err := ReadNoFollow(filepath.Join(dir, "missing"), 64); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("missing: err = %v", err) + } + if os.Getuid() != 0 { + if err := os.Chmod(p, 0); err != nil { + t.Fatal(err) + } + if _, err := ReadNoFollow(p, 64); !errors.Is(err, fs.ErrPermission) { + t.Errorf("unreadable: err = %v, want ErrPermission", err) + } + } +} + +func TestChmod(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + prev, changed, err := Chmod(path, 0o660) + if err != nil || !changed || prev != 0o600 { + t.Fatalf("Chmod = (%o, %v, %v)", prev, changed, err) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o660 { + t.Fatalf("mode = %v", info.Mode()) + } + if _, changed, err := Chmod(path, 0o660); err != nil || changed { + t.Fatalf("already matching: changed=%v err=%v", changed, err) + } + if _, _, err := Chmod(filepath.Join(dir, "absent"), 0o660); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("absent: %v", err) + } + + victim := filepath.Join(dir, "victim") + if err := os.WriteFile(victim, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link") + if err := os.Symlink(victim, link); err != nil { + t.Fatal(err) + } + if _, _, err := Chmod(link, 0o666); !errors.Is(err, ErrNotRegular) { + t.Fatalf("symlink: %v, want ErrNotRegular", err) + } + if info, _ := os.Stat(victim); info.Mode().Perm() != 0o600 { + t.Fatalf("victim mode changed through the symlink: %v", info.Mode()) + } + if _, _, err := Chmod(dir, 0o700); !errors.Is(err, ErrNotRegular) { + t.Fatalf("directory: %v, want ErrNotRegular", err) + } +} + +func TestCheckLocation(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + if info, err := CheckLocation(target); err != nil || info != nil { + t.Fatalf("absent: (%v, %v)", info, err) + } + if err := os.WriteFile(target, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if info, err := CheckLocation(target); err != nil || info == nil || !info.Mode().IsRegular() { + t.Fatalf("regular: (%v, %v)", info, err) + } + if os.Geteuid() != 0 { + // A directory that cannot be searched: the target's Lstat fails with + // something other than "not exist", and that is refused too. + closed := filepath.Join(dir, "closed") + if err := os.Mkdir(closed, 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(closed, 0o700) }) + if _, err := CheckLocation(filepath.Join(closed, "f")); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("unsearchable dir: %v", err) + } + } +} + +func TestCreateTemp_RandomFailures(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + old := randRead + t.Cleanup(func() { randRead = old }) + + randRead = func([]byte) (int, error) { return 0, errors.New("no entropy") } + if _, _, err := createTemp(target); err == nil || !strings.Contains(err.Error(), "no entropy") { + t.Errorf("entropy failure: %v", err) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers: %v", left) + } + + // Every name the generator can produce is already taken: refused after + // createAttempts, never written through the existing file. + randRead = func(b []byte) (int, error) { + for i := range b { + b[i] = 0xAB + } + return len(b), nil + } + taken := target + "." + strings.Repeat("ab", 8) + ".tmp" + if err := os.WriteFile(taken, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := Replace(target, []byte("new"), 0o600, Options{}); err == nil || !strings.Contains(err.Error(), "no free temporary name") { + t.Errorf("all names taken: %v", err) + } + if got, _ := os.ReadFile(taken); string(got) != "keep" { + t.Errorf("the taken name was written through: %q", got) + } + if _, err := os.Lstat(target); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("target appeared: %v", err) + } +} + +func TestReplace_FailuresLeaveNoTemp(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root is not stopped by directory modes") + } + old := chownFile + t.Cleanup(func() { chownFile = old }) + + // Cannot create the temporary: the directory is read-only. + ro := t.TempDir() + if err := os.Chmod(ro, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(ro, 0o700) }) + if err := Replace(filepath.Join(ro, "f"), []byte("x"), 0o600, Options{}); err == nil || errors.Is(err, ErrUnsafeLocation) { + t.Errorf("read-only dir: err = %v, want a create error", err) + } + + // The written file fails verification (mode changed under us). + dir := t.TempDir() + target := filepath.Join(dir, "f") + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + chownFile = func(f *os.File, _, _ int) error { return f.Chmod(0o644) } + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); err == nil || !strings.Contains(err.Error(), "written file is mode") { + t.Errorf("mode drift: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target replaced despite failed verification: %q", got) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers after failed fill: %v", left) + } + + // The flush fails: no rename, no leftover. + oldSync := syncFile + syncFile = func(*os.File) error { return errors.New("EIO") } + err := Replace(target, []byte("new"), 0o660, Options{}) + syncFile = oldSync + if err == nil || !strings.Contains(err.Error(), "EIO") { + t.Errorf("sync failure: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target replaced despite failed sync: %q", got) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers after failed sync: %v", left) + } + + // The temporary gained a second name while being written. + chownFile = func(f *os.File, _, _ int) error { + hl := f.Name() + ".hl" + t.Cleanup(func() { _ = os.Remove(hl) }) + return os.Link(f.Name(), hl) + } + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); err == nil || !strings.Contains(err.Error(), "links, want 1") { + t.Errorf("extra link: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target replaced despite extra link: %q", got) + } + + // The location changes while the temporary is being written. + chownFile = func(*os.File, int, int) error { + _ = os.Remove(target) + return os.Symlink("/nonexistent", target) + } + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("location changed mid-write: %v", err) + } + if info, err := os.Lstat(target); err != nil || info.Mode()&fs.ModeSymlink == 0 { + t.Errorf("the planted link was replaced: %v %v", info, err) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers after refused rename: %v", left) + } + _ = os.Remove(target) + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + + // The rename itself fails: the directory went read-only after the + // temporary was created. + chownFile = func(*os.File, int, int) error { return os.Chmod(dir, 0o500) } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + err = Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}) + _ = os.Chmod(dir, 0o700) + if err == nil || errors.Is(err, ErrUnsafeLocation) { + t.Errorf("rename failure: err = %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target changed on failed rename: %q", got) + } + // The temporary could not be removed from the read-only directory at + // the time; it is the one leftover this path can produce, and it is + // removable now. + for _, l := range leftovers(t, target) { + _ = os.Remove(l) + } +} diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index b2f04031b4..66e68db376 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -150,7 +150,9 @@ hive_run_uid_isolation_migration() { else echo "[entrypoint] UID isolation: repairing ${_agent_name} (uid ${_agent_uid}) in background" hive_reown_agent_paths_if_needed "$_agent_name" "$_agent_uid" - chmod g+rwX "$_agent_dir" 2>/dev/null || true + if [ -d "$_agent_dir" ] && ! [ -L "$_agent_dir" ]; then + chmod g+rwX "$_agent_dir" 2>/dev/null || true + fi hive_publish_uid_isolation_marker "$_agent_marker" "$_agent_expected" || true echo "[entrypoint] UID isolation: ${_agent_name} pass complete" fi @@ -685,6 +687,9 @@ hive_sweep_root_phase_paths() { _sweep_failed="" for _p in $HIVE_DATA_ROOT_PHASE_PATHS; do [ -e "$_p" ] || continue + # Never through a link: chown without -h acts on what the link names, and + # /data/home is writable by every agent uid. + [ -L "$_p" ] && continue # Already owned by the runtime user — the steady state on almost every # boot. Skip the syscall entirely so the common path costs one stat. [ "$(stat -c '%u' "$_p" 2>/dev/null || echo)" = "1001" ] && continue @@ -1421,6 +1426,9 @@ if [ "$(id -u)" = "0" ]; then fi for _seeded_dir in /data/agents/*/; do [ -d "$_seeded_dir" ] || continue + # The trailing-slash glob also matches a link to a directory; a + # recursive chown/chmod must never start from one. + [ -L "${_seeded_dir%/}" ] && continue if [ "$(stat -c '%u' "$_seeded_dir" 2>/dev/null || echo 0)" = "0" ]; then chown -R dev:node "$_seeded_dir" 2>/dev/null || true chmod -R g+rwX "$_seeded_dir" 2>/dev/null || true @@ -1489,6 +1497,27 @@ if [ "$(id -u)" = "0" ]; then chown dev:node /data/home/.copilot 2>/dev/null || true chmod 2775 /data/home/.claude /data/home/.claude/session-env 2>/dev/null || true chown dev:node /data/home/.claude /data/home/.claude/session-env 2>/dev/null || true + # hive_shared_dir_sticky DIR... — add the sticky bit to the shared CLI + # directories whose files the root-run perm guard and the in-process Go + # writers rename over: in a group-writable directory only the sticky bit + # keeps a dev-owned entry from being removed or replaced by another uid. + # Under the per-agent home layout every CLI rewrites its own copy inside + # /data/home/agents/, so nothing legitimate renames over a dev-owned + # file here. The legacy whole-directory layout (HIVE_SHARED_AGENT_HOME=1) + # does — each agent uid refreshes the shared file in place — so the bit is + # left off there. Group bits are untouched; the existing 2770/2775 stays. + # Only directories that are real directories get it: a chmod through a + # symlink would land on whatever the link names. + hive_shared_dir_sticky() { + [ "${HIVE_SHARED_AGENT_HOME:-0}" = "1" ] && return 0 + for _d; do + if [ -d "$_d" ] && ! [ -L "$_d" ]; then + chmod +t "$_d" 2>/dev/null || true + fi + done + return 0 + } + hive_shared_dir_sticky /data/home /data/home/.claude /data/home/.copilot # Codex (newer method) writes a local sqlite state db under $HOME/.codex — # pre-create it group-writable + setgid so every agent UID (node group) can # init it, matching .claude/.copilot. Without this, switching an agent to the @@ -1536,6 +1565,9 @@ if [ "$(id -u)" = "0" ]; then # process touches .bob first. 2770: no world access, these hold auth settings. chmod 2770 /data/home/.bob /data/home/.bob/settings 2>/dev/null || true chown dev:node /data/home/.bob /data/home/.bob/settings 2>/dev/null || true + # Not antigravity-cli/ or its cache/: those are bridged into each agent home + # as directories, so agy renames its token there under the agent's own uid. + hive_shared_dir_sticky /data/home/.gemini /data/home/.bob ln -sfn /data/config/github-copilot /home/dev/.config/github-copilot ln -sfn /data/config/github-copilot /data/home/.config/github-copilot ln -sfn /data/home/.copilot /home/dev/.copilot @@ -1625,18 +1657,52 @@ if [ "$(id -u)" = "0" ]; then # this is an OAuth token. The chown restores dev ownership as well, because # that is the invariant the in-process Go permissions watcher depends on — it # runs as dev after the privilege drop and can only chmod what dev owns. + # + # Both single-file repairs go through hive_fix_shared_file, which acts only + # on a regular file inside a regular directory: chmod and chown (without -h) + # act on whatever a path resolves to, and these directories are writable by + # every agent uid. The invariant is that a root-run repair never changes the + # mode or owner of anything an agent uid could have pointed the path at. hive_fix_shared_credential() { - [ -f "$1" ] || return 0 - chmod g+r "$1" 2>/dev/null || true - chown dev:node "$1" 2>/dev/null || true + hive_fix_shared_file "$1" g+r return 0 } # hive_fix_copilot_config — copilot rewrites config.json 0600 on refresh. hive_fix_copilot_config() { - [ -f /data/home/.copilot/config.json ] || return 0 - chmod 660 /data/home/.copilot/config.json 2>/dev/null || true - chown dev:node /data/home/.copilot/config.json 2>/dev/null || true + hive_fix_shared_file /data/home/.copilot/config.json 660 + return 0 + } + + # hive_fix_shared_file PATH MODE — chmod MODE and chown dev:node the regular + # file at PATH. Nothing is done when PATH is absent. When PATH or its + # directory is anything other than a regular file in a regular directory, + # nothing is done either, and a WARN is logged once per path per guard + # process (HIVE_PERM_GUARD_REFUSED is that process's memory of it). + # + # The `[ -L ]` tests are the decision; the find runs the chmod and chown + # from the same enumeration (`-type f` never matches a link) and `chown -h` + # never follows, so the time between the check and each action is as short + # as the shell can make it. The sticky bit set on the shared dot-dirs at + # boot is what keeps a dev-owned entry from being swapped out from under it. + hive_fix_shared_file() { + _p="$1"; _m="$2" + _dir="${_p%/*}"; _base="${_p##*/}" + if [ -d "$_dir" ] && ! [ -L "$_dir" ] && [ -f "$_p" ] && ! [ -L "$_p" ]; then + find "$_dir" -maxdepth 1 -name "$_base" -type f ! -type l -exec chmod "$_m" {} + -exec chown -h dev:node {} + 2>/dev/null || true + return 0 + fi + # Absent (and not a dangling link): nothing to repair, nothing to say. + if ! [ -e "$_p" ] && ! [ -L "$_p" ]; then + return 0 + fi + case " ${HIVE_PERM_GUARD_REFUSED:-} " in + *" $_p "*) ;; + *) + HIVE_PERM_GUARD_REFUSED="${HIVE_PERM_GUARD_REFUSED:-} $_p" + echo "[entrypoint] WARN: perm guard: $_p is not a regular file in a regular directory; left untouched" >&2 + ;; + esac return 0 } @@ -1747,7 +1813,9 @@ if [ "$(id -u)" = "0" ]; then # sweep still happens, on the 5-minute cycle. hive_fix_claude_instant() { hive_fix_shared_credential /data/home/.claude/.credentials.json - chmod g+rwx /data/home/.claude 2>/dev/null || true + if [ -d /data/home/.claude ] && ! [ -L /data/home/.claude ]; then + chmod g+rwx /data/home/.claude 2>/dev/null || true + fi return 0 } @@ -1765,7 +1833,11 @@ if [ "$(id -u)" = "0" ]; then hive_fix_gemini_instant() { hive_fix_shared_credential /data/home/.gemini/antigravity-cli/antigravity-oauth-token hive_fix_shared_credential /data/home/.gemini/antigravity-cli/cache/onboarding.json - chmod g+rwx /data/home/.gemini /data/home/.gemini/antigravity-cli /data/home/.gemini/antigravity-cli/cache 2>/dev/null || true + for _d in /data/home/.gemini /data/home/.gemini/antigravity-cli /data/home/.gemini/antigravity-cli/cache; do + if [ -d "$_d" ] && ! [ -L "$_d" ]; then + chmod g+rwx "$_d" 2>/dev/null || true + fi + done return 0 } @@ -1898,6 +1970,11 @@ if [ "$(id -u)" = "0" ]; then # launch. Falling back to the proxy CA alone if the combined bundle isn't there # yet keeps proxied HTTPS working; the combined bundle is strictly safer because # it also trusts real public roots for any non-proxied endpoint (advisory mode). + # Root writes this file into a directory every agent uid can write to, so + # it is only ever written when the path is absent or already a regular file. + if [ -L /data/home/.bashrc ]; then + echo "[entrypoint] WARN: /data/home/.bashrc is not a regular file; left untouched" + else cat > /data/home/.bashrc <<'BASHRC' 2>/dev/null || true # Hive agent shell environment if [ -f /data/proxy-ca-bundle.pem ]; then @@ -1914,15 +1991,16 @@ BASHRC # Written by root via `cat >` above, so it is created root:root. Hand it to # dev at the point of creation (#5369): the recursive /data chown is guarded # off on every normal boot and cannot be relied on to fix it later. - chown dev:node /data/home/.bashrc 2>/dev/null || true + chown -h dev:node /data/home/.bashrc 2>/dev/null || true + fi # Login shells (tmux default-command) read ~/.profile, not ~/.bashrc — chain # them so both shell flavors get the same environment. - if [ ! -f /data/home/.profile ]; then + if [ ! -f /data/home/.profile ] && ! [ -L /data/home/.profile ]; then # shellcheck disable=SC2016 printf '[ -f "$HOME/.bashrc" ] && . "$HOME/.bashrc"\n' > /data/home/.profile 2>/dev/null || true chmod 644 /data/home/.profile 2>/dev/null || true # Same as .bashrc above — created by root, so hand it over here (#5369). - chown dev:node /data/home/.profile 2>/dev/null || true + chown -h dev:node /data/home/.profile 2>/dev/null || true fi # ── #5369: targeted post-phase ownership sweep ──────────────────────── @@ -2020,8 +2098,10 @@ BASHRC fi # Ensure agent dirs are group-writable so the dev user (same node group) # can clean up stale workspaces at runtime without root privileges. - chmod g+rwX "/data/agents/${agent_name}" 2>/dev/null || true - if [ -d "/data/beads/${agent_name}" ]; then + if [ -d "/data/agents/${agent_name}" ] && ! [ -L "/data/agents/${agent_name}" ]; then + chmod g+rwX "/data/agents/${agent_name}" 2>/dev/null || true + fi + if [ -d "/data/beads/${agent_name}" ] && ! [ -L "/data/beads/${agent_name}" ]; then BEAD_DIR_OWNER=$(stat -c '%u' "/data/beads/${agent_name}" 2>/dev/null || echo "0") if [ "$BEAD_DIR_OWNER" != "$AGENT_UID" ]; then chown -R "hive-${agent_name}:node" "/data/beads/${agent_name}" 2>/dev/null || true diff --git a/src/deploy/test_entrypoint_perm_guard.sh b/src/deploy/test_entrypoint_perm_guard.sh index 857dfce8dd..9b34f8ca07 100755 --- a/src/deploy/test_entrypoint_perm_guard.sh +++ b/src/deploy/test_entrypoint_perm_guard.sh @@ -594,6 +594,90 @@ for repair in hive_fix_tree_recent hive_fix_tree; do done rm -f "$SETTINGS" "$SIBLING" +# ── 14. The single-file repairs act only on a regular file in a regular dir ── +# chmod and chown (without -h) act on whatever a path resolves to. The shared +# dot-dirs are writable by every agent uid, so the file-level repairs must do +# nothing at all when the path, or the directory it is in, is not what the +# guard expects — and say so once, not once per 5-second poll. +mode_of() { stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1"; } +PLANT_HOME="$WORK/plant-home" +PLANT_OUT="$WORK/plant-outside" +mkdir -p "$PLANT_HOME/.claude" "$PLANT_HOME/.copilot" +sed "s#/data/home#${PLANT_HOME}#g" "$GUARDS" > "$WORK/guards.plant.sh" +printf 'outside\n' > "$PLANT_OUT" +chmod 0600 "$PLANT_OUT" + +# A link where the credential should be. +ln -s "$PLANT_OUT" "$PLANT_HOME/.claude/.credentials.json" +ln -s "$PLANT_OUT" "$PLANT_HOME/.copilot/config.json" +PLANT_LOG="$WORK/plant.log" +sh -c 'set -e; . "$1"; hive_fix_credentials_fast; hive_fix_credentials_fast; hive_fix_claude_instant' \ + sh "$WORK/guards.plant.sh" >/dev/null 2>"$PLANT_LOG" +if [ "$(mode_of "$PLANT_OUT")" = "600" ]; then + ok "a link planted at the credential path is not acted on (target stays 0600)" +else + bad "a link planted at the credential path is not acted on" \ + "target went to mode $(mode_of "$PLANT_OUT")" + chmod 0600 "$PLANT_OUT" +fi +if [ -L "$PLANT_HOME/.claude/.credentials.json" ] && [ -L "$PLANT_HOME/.copilot/config.json" ]; then + ok "the planted links themselves are left in place" +else + bad "the planted links themselves are left in place" +fi +warn_count="$(grep -c 'WARN: perm guard' "$PLANT_LOG" || true)" +if [ "$warn_count" = "2" ]; then + ok "a refused path is reported once per guard process (2 paths, 3 passes, 2 WARNs)" +else + bad "a refused path is reported once per guard process" \ + "expected 2 WARN lines, got $warn_count: $(cat "$PLANT_LOG")" +fi +rm -f "$PLANT_HOME/.claude/.credentials.json" "$PLANT_HOME/.copilot/config.json" + +# A link where the directory should be, with a real file behind it. +rmdir "$PLANT_HOME/.copilot" +mkdir -p "$WORK/plant-elsewhere" +printf '{}\n' > "$WORK/plant-elsewhere/config.json" +chmod 0600 "$WORK/plant-elsewhere/config.json" +ln -s "$WORK/plant-elsewhere" "$PLANT_HOME/.copilot" +sh -c 'set -e; . "$1"; hive_fix_copilot_config' sh "$WORK/guards.plant.sh" >/dev/null 2>&1 +if [ "$(mode_of "$WORK/plant-elsewhere/config.json")" = "600" ]; then + ok "a link planted at the directory is not acted through (file behind it stays 0600)" +else + bad "a link planted at the directory is not acted through" \ + "file behind the link went to mode $(mode_of "$WORK/plant-elsewhere/config.json")" +fi +rm -f "$PLANT_HOME/.copilot" +mkdir -p "$PLANT_HOME/.copilot" + +# A regular file in a regular directory is still repaired, and an absent one +# is silent. +printf '{}\n' > "$PLANT_HOME/.copilot/config.json" +chmod 0600 "$PLANT_HOME/.copilot/config.json" +sh -c 'set -e; . "$1"; hive_fix_copilot_config' sh "$WORK/guards.plant.sh" >/dev/null 2>"$PLANT_LOG" +if group_rw "$(mode_of "$PLANT_HOME/.copilot/config.json")"; then + ok "a regular config.json in a regular .copilot is still reopened (mode $(mode_of "$PLANT_HOME/.copilot/config.json"))" +else + bad "a regular config.json in a regular .copilot is still reopened" \ + "mode $(mode_of "$PLANT_HOME/.copilot/config.json")" +fi +rm -f "$PLANT_HOME/.copilot/config.json" +sh -c 'set -e; . "$1"; hive_fix_credentials_fast' sh "$WORK/guards.plant.sh" >/dev/null 2>>"$PLANT_LOG" +if [ -s "$PLANT_LOG" ]; then + bad "an absent shared file is silent" "$(cat "$PLANT_LOG")" +else + ok "an absent shared file is silent" +fi + +# The guard that survives a planted link also survives the next loop: the +# refusal returns 0 under set -e. +if sh -c 'set -e; . "$1"; ln -s "$2" "$3/.claude/.credentials.json"; hive_fix_shared_credential "$3/.claude/.credentials.json"; echo alive' \ + sh "$WORK/guards.plant.sh" "$PLANT_OUT" "$PLANT_HOME" 2>/dev/null | grep -q alive; then + ok "a refusal does not end the guard under set -e" +else + bad "a refusal does not end the guard under set -e" +fi + echo "" echo "Results: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] || exit 1 diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 02f8baaaf1..e270605f8b 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -633,6 +633,20 @@ Each per-UID agent runs with its own durable `HOME` under `/data/home/agents//memory/MEMORY.md`, per-agent by path) was writable by every other agent and persisted on the PVC forever. Provisioning now retires those whole-directory bridges on the next launch, keeps only the login/settings entries above, and never deletes or moves anything under `/data/home`: a pre-existing shared memory store is left in place and logged once (`shared CLI home still holds a persistent memory store …`) so you can review or drain it ([#17033](https://github.com/hivecommons/hive/issues/17033)) before removing it by hand. The dashboard's token/cost scanners (`data.claude_sessions_dir`, `data.bob_sessions_dir`, `data.copilot_sessions_dir`) walk the shared path **and** every `/data/home/agents/*/…` sibling, attributing each per-agent tree to its owner, so cost accounting does not regress. `HIVE_SHARED_AGENT_HOME=1` restores the legacy single shared `HOME` for every agent (and with it the shared memory). +#### How the Claude login stays shared when the CLI replaces the link + +Claude Code saves a refreshed OAuth token (about every 8 hours) by writing a temp file in `~/.claude` and renaming it over `.credentials.json`. A rename replaces a symlink rather than writing through it, so after the first refresh the agent is running on a private `0600` copy of the token, the refresh has rotated the grant the shared file still holds, and every other agent fails its own refresh. On per-UID hives this logged the whole fleet out about every 8 hours ([#17039](https://github.com/hivecommons/hive/pull/17039) follow-up). + +Hive now treats the login as the one piece of `.claude` state that *is* shared, and publishes it back: at every agent launch, every `HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL` (default `5s`) from the credential watchdog, and right after a dashboard sign-in, it checks each claude agent's `~/.claude/.credentials.json`. When a regular file has replaced the link, hive reads it — directly, or as the agent's own uid through the same `su-exec` seam that adopts `.claude.json` sessions, so an agent-owned `0600` file is no obstacle — and: + +- keeps whichever credential is newer **and** usable in the shared `/data/home/.claude/.credentials.json`: the agent's copy wins only when it is complete (access token, refresh token, recorded `expiresAt`), usable, and expires later than the shared one, or the shared one is unusable. A failed refresh (empty tokens, `expiresAt` 0) never wins; +- moves the private file aside to `.credentials.json.stale-` in the agent's `.claude` (owner and mode unchanged) and, when the agent's copy was promoted, preserves the old shared file beside the shared credential the same way. Nothing is deleted; remove the `.stale-*` files by hand once you no longer want them; +- restores the symlink and records a `claude_credential_rebridged` audit entry (`kept=agent|shared`, never token material). + +When the file cannot be read either way (a deployment without `su-exec`), nothing is moved or relinked, the agent keeps the token it already had, and one `ERROR` per agent and file version names the path. The one exception is a dashboard sign-in: a fresh shared login is newer than anything an unreadable private copy could hold, so that trigger moves the unreadable file aside unread and relinks the agent. Memory, transcripts and todos stay per agent throughout; only the credential file is published. + +Every hive-side write to a shared login or CLI config (`.claude/.credentials.json`, `.copilot/config.json`, the durable Copilot token) goes through `crux/sharedfile`: a randomly named temporary beside the target, mode and owner set on the open descriptor, verified, then renamed into place, and only when the target is absent or a regular file inside a regular directory the server owns. Anything else at those paths is refused and logged once. The entrypoint's root permission guard applies the same rule to the files it repairs, and the shared dot-dirs carry the sticky bit so a dev-owned entry cannot be replaced by another uid. + ### CLI auto-memory (`agents.cli_memory`) Subscription CLIs keep their own long-lived "memory" next to the transcripts: Claude Code's auto-memory (`~/.claude/projects/-data-agents-/memory/MEMORY.md`), Codex memories (`$CODEX_HOME/memories/`), Copilot CLI's `memory` setting, and Gemini's `~/.gemini/GEMINI.md`. Hive agents already get their durable context from beads, the knowledge store, and the kick prompt, so a hidden per-CLI notebook mostly accumulates stale one-off inferences ([#17032](https://github.com/hivecommons/hive/issues/17032)). The `agents.cli_memory` block controls it fleet-wide, and each agent may override it with its own `cli_memory:` block: diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index c97f5fc7dd..fd410f8a76 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -311,6 +311,7 @@ and [network/credential controls](security-model.md#5-network-and-credentials). | `CODEX_API_KEY` | No | none | API key read by `pkg/agent/authprobe.go` for the Codex CLI backend; either this or `OPENAI_API_KEY` makes Codex API-key mode count as configured. | | `HIVE_AGENT_TOKEN_REFRESH_INTERVAL` | No | `40m` | Go duration overriding the per-agent token refresh interval. Invalid or non-positive values fall back to the default. | | `HIVE_CREDENTIAL_WATCHDOG_INTERVAL` | No | `5m` | Go duration overriding how often the credential watchdog verifies each in-use backend credential file. `0` does NOT disable the watchdog — disabling is intentionally not offered. | +| `HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL` | No | `5s` | Go duration overriding how often the credential watchdog checks each per-UID claude agent's `~/.claude/.credentials.json` link for a token refresh that replaced it with a private file, and publishes that refresh to the shared login (see [Per-agent homes](agent-configuration.md)). One `lstat` per claude agent per tick. | | `HIVE_PROVIDER_ERROR_BACKOFF_BASE` | No | `2m` | Go duration for the first inference-provider error backoff before another kick is allowed. Invalid or non-positive values fall back to the default. | | `HIVE_PROVIDER_ERROR_BACKOFF_MAX` | No | `30m` | Go duration cap for exponential inference-provider error backoff. Invalid or non-positive values fall back to the default. | | `HIVE_START_FAILURE_BLOCK_THRESHOLD` | No | `3` | Consecutive identical start failures (positive integer) before an agent is blocked from automatic relaunch (`pkg/agent/start_failure.go`). Backoff is applied from the first failure regardless; invalid values fall back to the default. | diff --git a/src/pkg/dashboard/audit.go b/src/pkg/dashboard/audit.go index af149dfa61..3e4a1e1a52 100644 --- a/src/pkg/dashboard/audit.go +++ b/src/pkg/dashboard/audit.go @@ -56,6 +56,7 @@ var auditSensitiveActions = map[string]bool{ "claude_auth_complete": true, "claude_auth_logout": true, "claude_auth_start": true, + "claude_credential_rebridged": true, "claude_token_missing": true, "config_agent_cadences": true, "config_agent_channels": true, From c4b6a0e63c2cbf4c8fbb72b365e6f78cfc22add8 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 00:22:14 +0000 Subject: [PATCH 031/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.224.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.224.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 15 +++++++++++++++ changelog.d/added-benchmarks-export-cost.md | 1 - changelog.d/added-marketplace-curated-techdocs.md | 1 - changelog.d/fixed-marketplace-stream-tarball.md | 1 - .../security-17730-upstreamproxy-url-redact.md | 1 - 5 files changed, 15 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/added-benchmarks-export-cost.md delete mode 100644 changelog.d/added-marketplace-curated-techdocs.md delete mode 100644 changelog.d/fixed-marketplace-stream-tarball.md delete mode 100644 changelog.d/security-17730-upstreamproxy-url-redact.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 541d4ef742..9e157c27e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,21 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.224.0) + +### Added + +- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. +- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). + +### Fixed + +- Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. + +### Security + +- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). + ## 2026-11-11 (v5.223.0) ### Added diff --git a/changelog.d/added-benchmarks-export-cost.md b/changelog.d/added-benchmarks-export-cost.md deleted file mode 100644 index 62f95173f7..0000000000 --- a/changelog.d/added-benchmarks-export-cost.md +++ /dev/null @@ -1 +0,0 @@ -- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. diff --git a/changelog.d/added-marketplace-curated-techdocs.md b/changelog.d/added-marketplace-curated-techdocs.md deleted file mode 100644 index 4a22b467ee..0000000000 --- a/changelog.d/added-marketplace-curated-techdocs.md +++ /dev/null @@ -1 +0,0 @@ -- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). diff --git a/changelog.d/fixed-marketplace-stream-tarball.md b/changelog.d/fixed-marketplace-stream-tarball.md deleted file mode 100644 index fd6a16ff29..0000000000 --- a/changelog.d/fixed-marketplace-stream-tarball.md +++ /dev/null @@ -1 +0,0 @@ -- Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. diff --git a/changelog.d/security-17730-upstreamproxy-url-redact.md b/changelog.d/security-17730-upstreamproxy-url-redact.md deleted file mode 100644 index 432e2a58ef..0000000000 --- a/changelog.d/security-17730-upstreamproxy-url-redact.md +++ /dev/null @@ -1 +0,0 @@ -- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). From 37f64c3305e73f53d342bd651980282d4fc01686 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 17:23:38 -0700 Subject: [PATCH 032/223] =?UTF-8?q?=E2=9C=A8=20hub:=20per-cluster=20host?= =?UTF-8?q?=5Faliases=20for=20hosted=20hives=20(#17805)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub: per-cluster host_aliases for hosted hives Adds ClusterConfig.host_aliases ([{ip, hostnames[]}]) to /data/saas/clusters.json, validated at load (IP parses, hostnames non-empty, lowercased; invalid blocks are dropped with a warning, never the cluster). Rendered as Pod hostAliases in the hosted-hive Deployment template next to the upstream_proxy env injection, and a throttled, rate-limited reconcile sweep (3 patches / 15 min) converges existing hives while preserving unrelated hand-applied aliases. Needed for GHE hosts reachable only through a TLS reverse proxy on a fixed IP (no CONNECT proxy), where upstream_proxy cannot apply. Fixes #17801 Signed-off-by: Andrew Anderson * 📖 api-reference: resync hub-server.go citations after host_aliases field Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/added-hub-cluster-host-aliases.md | 1 + src/docs/api-reference/hub-server.md | 58 ++-- src/docs/hub-deployment.md | 2 + src/docs/network-requirements.md | 30 +++ src/pkg/hub/cluster_host_aliases.go | 247 ++++++++++++++++++ src/pkg/hub/cluster_host_aliases_test.go | 150 +++++++++++ src/pkg/hub/clusters_registry.go | 10 + src/pkg/hub/saas_provision.go | 20 ++ src/pkg/hub/saas_sha_poller.go | 6 + src/pkg/hub/server.go | 3 + 10 files changed, 498 insertions(+), 29 deletions(-) create mode 100644 changelog.d/added-hub-cluster-host-aliases.md create mode 100644 src/pkg/hub/cluster_host_aliases.go create mode 100644 src/pkg/hub/cluster_host_aliases_test.go diff --git a/changelog.d/added-hub-cluster-host-aliases.md b/changelog.d/added-hub-cluster-host-aliases.md new file mode 100644 index 0000000000..ebc6e54a5c --- /dev/null +++ b/changelog.d/added-hub-cluster-host-aliases.md @@ -0,0 +1 @@ +- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 0c4f7404d3..c8891dd864 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,42 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1789` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1790` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1792` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1793` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1721` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1722` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1723` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1724` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1725` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1736` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1737` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1738` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1739` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1724` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1725` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1726` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1727` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1728` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1739` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1740` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1741` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1742` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1747` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1757` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1758` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1759` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1760` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1761` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1762` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1763` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1770` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1771` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1778` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1779` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1780` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1802` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1804` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1750` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1760` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1761` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1762` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1763` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1764` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1765` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1766` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1773` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1783` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1800` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1805` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1807` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index f617e25ad2..fd842d5b06 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -101,6 +101,8 @@ For push-reported clusters where the hub cannot run `kubectl`, follow the manife ### Push-reported cluster entries +`/data/saas/clusters.json` also accepts per-cluster network overrides for hives it provisions: `upstream_proxy` (`{url, hosts[], no_proxy[]}`, a CONNECT forward proxy, injected as `HIVE_UPSTREAM_PROXY_*` env) and `host_aliases` (`[{ip, hostnames[]}]`, rendered as Pod `hostAliases` for a GHE reachable only via a TLS reverse proxy on a fixed IP; existing hives are converged by a rate-limited reconcile sweep). See [network-requirements.md](network-requirements.md). + `/data/saas/clusters.json` keeps the historical compatibility key `pull_only`. Set `pull_only: true` when the hub cannot reach that cluster's Kubernetes API and the spokes must dial out to the hub instead. Operator-facing surfaces call these clusters **push-reported** because their node stats, dashboard token proofs, and liveness arrive in spoke heartbeats pushed from the cluster. A push-reported entry still needs its stable identity (for example `id`, `name`, `domain`, app/forge fields), but it does not need a usable `kubeconfig_path` for hub-side reads. If node stats are absent, upgrade the spokes to a build that sends heartbeat cluster health and then check the error reason shown in the Clusters panel. diff --git a/src/docs/network-requirements.md b/src/docs/network-requirements.md index 198496a7ed..ddb1af9e3e 100644 --- a/src/docs/network-requirements.md +++ b/src/docs/network-requirements.md @@ -97,6 +97,36 @@ slots on that cluster are provisioned with the env vars above: Existing hive Deployments are not reconciled; patch them once with `kubectl -n set env deploy/hive -c hive HIVE_UPSTREAM_PROXY_URL=... HIVE_UPSTREAM_PROXY_HOSTS=...`. +### GitHub Enterprise behind a TLS reverse proxy (hostAliases) + +Some networks expose an enterprise GitHub not through a CONNECT forward proxy +but through a **TLS reverse proxy** pinned to a fixed IP: the proxy serves a +certificate valid for the GHE hostnames on :443 and forwards to GHE, while +`CONNECT` is rejected (e.g. openresty answering `405 Not Allowed`) and direct +egress to GHE times out. `upstream_proxy` cannot express that; the fix is a +Pod `hostAliases` entry that resolves the GHE hostnames to the proxy IP so TLS +SNI/verification and every path (API, git smart-HTTP, MITM) keep working. + +Hosted hubs declare it per cluster in `/data/saas/clusters.json`: + +```json +{ + "id": "corp-cluster", + "host_aliases": [ + { "ip": "129.34.104.21", "hostnames": ["github.ibm.com", "api.github.ibm.com"] } + ] +} +``` + +Entries are validated at load (the IP must parse, every entry needs at least +one hostname; hostnames are lowercased). An invalid `host_aliases` block is +logged and ignored — it never drops the cluster. New pool slots on the cluster +are provisioned with the matching `spec.template.spec.hostAliases`, and a +throttled reconcile sweep (at most 3 patches per 15 minutes, since each one +rolls that hive's pod) converges existing hives. Unrelated hand-applied +aliases for other IPs are preserved. Non-hub (standalone) deployments add the +same `hostAliases` block to their own pod spec. + ## Firewall recommendations - Public edge: expose only the dashboard/hub front door (3001 for Docker/hub, or your HTTPS Ingress/Route for Kubernetes). Keep 3002 and 7681 cluster/private unless you have an explicit operator access path. diff --git a/src/pkg/hub/cluster_host_aliases.go b/src/pkg/hub/cluster_host_aliases.go new file mode 100644 index 0000000000..d3940710a3 --- /dev/null +++ b/src/pkg/hub/cluster_host_aliases.go @@ -0,0 +1,247 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sort" + "strings" + "time" +) + +const ( + // hostAliasesReconcileInterval throttles the sweep. Like NET_ADMIN and the + // per-hive env lane this is static remediation: a hive either carries the + // cluster's hostAliases or it does not, and a converged hive stays + // converged until clusters.json changes. + hostAliasesReconcileInterval = 15 * time.Minute + + // hostAliasesKubectlTimeout bounds each per-hive kubectl get/patch so one + // unreachable cluster cannot stall the whole sweep. + hostAliasesKubectlTimeout = 15 * time.Second + + // hostAliasesMaxPatchesPerCycle caps PATCHES per sweep. A hostAliases change + // is a podspec change and rolls the pod; the 2026-11-11 vllm-d rollout + // showed 28 simultaneous rollouts saturating CRI-O on one worker (5.8 GB + // image) and liveness-killing neighbours, so convergence is deliberately + // slow: 3 per 15 min. + hostAliasesMaxPatchesPerCycle = 3 + + // podHostAliasesPath is the JSON-Patch path to the pod-level hostAliases + // list in the hive Deployment. + podHostAliasesPath = "/spec/template/spec/hostAliases" +) + +// ClusterHostAlias is one Pod hostAliases entry the hosted-hive provisioner +// renders into every hive Deployment on the cluster (and the reconcile sweep +// converges onto existing ones). It exists for GitHub Enterprise hosts that +// are reachable only through a TLS REVERSE proxy pinned to a fixed IP — not a +// CONNECT forward proxy, so upstream_proxy cannot express it (#17801). +type ClusterHostAlias struct { + IP string `json:"ip" yaml:"ip"` + Hostnames []string `json:"hostnames" yaml:"hostnames"` +} + +// normalizeClusterHostAliases validates and canonicalises a cluster's +// host_aliases: the IP must parse, every entry needs at least one hostname, +// hostnames are trimmed and lowercased, blanks dropped. Returns nil for an +// empty input so callers can test `len(out) == 0`. +func normalizeClusterHostAliases(in []ClusterHostAlias) ([]ClusterHostAlias, error) { + if len(in) == 0 { + return nil, nil + } + out := make([]ClusterHostAlias, 0, len(in)) + for i, a := range in { + ip := strings.TrimSpace(a.IP) + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("host_aliases[%d]: ip %q does not parse", i, a.IP) + } + names := make([]string, 0, len(a.Hostnames)) + for _, h := range a.Hostnames { + h = strings.ToLower(strings.TrimSpace(h)) + if h == "" { + continue + } + if strings.ContainsAny(h, " \t/:") { + return nil, fmt.Errorf("host_aliases[%d]: hostname %q is not a bare host", i, h) + } + names = append(names, h) + } + if len(names) == 0 { + return nil, fmt.Errorf("host_aliases[%d]: ip %s has no hostnames", i, ip) + } + out = append(out, ClusterHostAlias{IP: ip, Hostnames: names}) + } + return out, nil +} + +// hostAliasesPatchJSON builds the JSON-Patch that sets the pod-level +// hostAliases to exactly want. "add" on an existing path replaces it, so one +// op serves both the missing and the drifted case. +func hostAliasesPatchJSON(want []ClusterHostAlias) (string, error) { + val, err := json.Marshal(want) + if err != nil { + return "", err + } + patch, err := json.Marshal([]map[string]any{{ + "op": "add", "path": podHostAliasesPath, "value": json.RawMessage(val), + }}) + if err != nil { + return "", err + } + return string(patch), nil +} + +// hostAliasesConverged reports whether the live hostAliases (raw jsonpath +// output, possibly empty) already carry every entry in want. Order and +// hostname case are ignored; EXTRA live entries are tolerated so an operator's +// hand-applied alias for an unrelated host is never clobbered by the sweep. +func hostAliasesConverged(liveRaw []byte, want []ClusterHostAlias) bool { + if len(want) == 0 { + return true + } + var live []ClusterHostAlias + if len(strings.TrimSpace(string(liveRaw))) > 0 { + if err := json.Unmarshal(liveRaw, &live); err != nil { + return false + } + } + liveSet := map[string]bool{} + for _, a := range live { + for _, h := range a.Hostnames { + liveSet[strings.TrimSpace(a.IP)+"|"+strings.ToLower(strings.TrimSpace(h))] = true + } + } + for _, a := range want { + for _, h := range a.Hostnames { + if !liveSet[a.IP+"|"+h] { + return false + } + } + } + return true +} + +// mergeHostAliases returns want merged over live: live entries for IPs not in +// want are kept (so unrelated hand-applied aliases survive), entries for IPs +// in want are replaced by the desired hostnames. Output is sorted by IP so the +// patch is deterministic. +func mergeHostAliases(liveRaw []byte, want []ClusterHostAlias) []ClusterHostAlias { + var live []ClusterHostAlias + if len(strings.TrimSpace(string(liveRaw))) > 0 { + _ = json.Unmarshal(liveRaw, &live) + } + wantIPs := map[string]bool{} + for _, a := range want { + wantIPs[a.IP] = true + } + out := make([]ClusterHostAlias, 0, len(live)+len(want)) + for _, a := range live { + if !wantIPs[strings.TrimSpace(a.IP)] { + out = append(out, a) + } + } + out = append(out, want...) + sort.SliceStable(out, func(i, j int) bool { return out[i].IP < out[j].IP }) + return out +} + +// reconcileHostAliasesIfDue runs the sweep at most once per +// hostAliasesReconcileInterval. Safe to call from the poller loop every tick; +// same guarding mutex as reconcileNetAdminIfDue (poller-loop-only state). +func (s *HubServer) reconcileHostAliasesIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastHostAliasesReconcile.IsZero() || + time.Since(s.lastHostAliasesReconcile) >= hostAliasesReconcileInterval + if due { + s.lastHostAliasesReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileHostAliases() +} + +// reconcileHostAliases sweeps every hosted hive whose cluster declares +// host_aliases and patches the Deployment's pod hostAliases where the desired +// entries are absent — at most hostAliasesMaxPatchesPerCycle per cycle, since +// every patch rolls that hive's pod. Clusters WITHOUT host_aliases are never +// touched, so the lane cannot strip anything an operator applied by hand. +func (s *HubServer) reconcileHostAliases() { + wantByCluster := map[string][]ClusterHostAlias{} + for id, c := range s.clusters { + if len(c.HostAliases) > 0 { + wantByCluster[id] = c.HostAliases + } + } + if len(wantByCluster) == 0 { + return + } + + patched := 0 + deferred := 0 + for _, h := range listSaaSHives() { + if !netAdminSweepEligible(h.Status) { + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil { + continue + } + want, ok := wantByCluster[cluster.ID] + if !ok || s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + ns := hiveHostedNamespacePrefix + h.ID + + ctx, cancel := context.WithTimeout(context.Background(), hostAliasesKubectlTimeout) + getCmd := kubectlForClusterContext(ctx, cluster, "get", "deployment", "hive", + "-n", ns, "-o", "jsonpath={"+strings.ReplaceAll(podHostAliasesPath, "/", ".")+"}") + out, err := getCmd.Output() + cancel() + if err != nil { + s.logger.Debug("host_aliases reconcile: could not read hive deployment", + "hive_id", h.ID, "cluster", cluster.ID, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + if hostAliasesConverged(out, want) { + continue + } + if patched >= hostAliasesMaxPatchesPerCycle { + deferred++ + continue + } + patch, perr := hostAliasesPatchJSON(mergeHostAliases(out, want)) + if perr != nil { + s.logger.Warn("host_aliases reconcile: could not build patch", + "hive_id", h.ID, "cluster", cluster.ID, "error", perr) + continue + } + pctx, pcancel := context.WithTimeout(context.Background(), hostAliasesKubectlTimeout) + patchCmd := kubectlForClusterContext(pctx, cluster, "patch", "deployment", "hive", + "-n", ns, "--type=json", "-p", patch) + pout, err := patchCmd.CombinedOutput() + pcancel() + if err != nil { + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("host_aliases reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, + "output", strings.TrimSpace(string(pout)), "error", err) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled cluster host_aliases onto hive deployment (rolls the pod once)", + "hive_id", h.ID, "cluster", cluster.ID, "patched_this_cycle", patched) + } + if deferred > 0 { + s.logger.Info("host_aliases reconcile: rate limit reached, remaining hives deferred to next cycle", + "patched", patched, "deferred", deferred, + "max_per_cycle", hostAliasesMaxPatchesPerCycle, + "next_cycle_in", hostAliasesReconcileInterval.String()) + } +} diff --git a/src/pkg/hub/cluster_host_aliases_test.go b/src/pkg/hub/cluster_host_aliases_test.go new file mode 100644 index 0000000000..cbb502f462 --- /dev/null +++ b/src/pkg/hub/cluster_host_aliases_test.go @@ -0,0 +1,150 @@ +package hub + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + "text/template" +) + +func renderManifestWithHostAliases(t *testing.T, aliases []ClusterHostAlias) string { + t.Helper() + tmpl, err := template.New("manifests").Parse(k8sManifestTemplate) + if err != nil { + t.Fatalf("template does not parse: %v", err) + } + data := map[string]any{ + "ID": "test-hive", "ImageTag": "v2-latest", "ImagePullPolicy": "Always", + "InitContainerUID": "1001", "InitContainerGID": "1000", + "TerminalPort": 7681, "DashboardPort": 8080, + "CPURequest": "500m", "MemRequest": "1Gi", "CPULimit": "2", "MemLimit": "4Gi", + "Namespace": "hive-test", "StorageSize": "10Gi", "StorageClass": "standard", + "HeartbeatKey": "hb", "SessionKey": "sk", "SSOPublicKey": "pk", "TerminalKey": "tk", + "AuthorizedUsers": "alice:owner", "Host": "test-hive.example", "Owner": "alice", + "HubPublicURL": "https://hub.example", "ProjectName": "test", "PrimaryRepo": "org/repo", + "AppKeys": []provisionAppKey{}, + "HostAliases": aliases, + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + t.Fatalf("template does not execute: %v", err) + } + return buf.String() +} + +func TestClusterHostAliasesRenderedIntoPodSpec(t *testing.T) { + m := renderManifestWithHostAliases(t, []ClusterHostAlias{ + {IP: "129.34.104.21", Hostnames: []string{"github.ibm.com", "api.github.ibm.com"}}, + }) + want := " spec:\n hostAliases:\n - ip: \"129.34.104.21\"\n hostnames:\n - \"github.ibm.com\"\n - \"api.github.ibm.com\"\n" + // The template carries a comment line between spec: and hostAliases:. + stripped := strings.Join(func() []string { + var out []string + for _, l := range strings.Split(m, "\n") { + if strings.HasPrefix(strings.TrimSpace(l), "#") { + continue + } + out = append(out, l) + } + return out + }(), "\n") + if !strings.Contains(stripped, want) { + t.Fatalf("rendered manifest missing pod hostAliases block:\n%s\n--- got ---\n%s", want, m) + } +} + +func TestClusterHostAliasesOmittedWhenUnset(t *testing.T) { + for name, aliases := range map[string][]ClusterHostAlias{"nil": nil, "empty": {}} { + if m := renderManifestWithHostAliases(t, aliases); strings.Contains(m, "hostAliases") { + t.Errorf("%s: hostAliases must not be rendered when no aliases configured", name) + } + } +} + +func TestNormalizeClusterHostAliases(t *testing.T) { + got, err := normalizeClusterHostAliases([]ClusterHostAlias{ + {IP: " 129.34.104.21 ", Hostnames: []string{" GitHub.IBM.com ", "", "api.github.ibm.com"}}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 1 || got[0].IP != "129.34.104.21" || + strings.Join(got[0].Hostnames, ",") != "github.ibm.com,api.github.ibm.com" { + t.Fatalf("unexpected normalization: %+v", got) + } + if out, err := normalizeClusterHostAliases(nil); err != nil || out != nil { + t.Fatalf("nil input should yield nil,nil; got %v,%v", out, err) + } + for name, in := range map[string][]ClusterHostAlias{ + "bad ip": {{IP: "not-an-ip", Hostnames: []string{"a.example"}}}, + "no hostnames": {{IP: "10.0.0.1", Hostnames: []string{" ", ""}}}, + "url not host": {{IP: "10.0.0.1", Hostnames: []string{"https://a.example"}}}, + } { + if _, err := normalizeClusterHostAliases(in); err == nil { + t.Errorf("%s: expected error", name) + } + } +} + +func TestValidateClusterEntriesKeepsClusterOnBadHostAliases(t *testing.T) { + logger := slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)) + got := validateClusterEntries([]ClusterConfig{ + {ID: "bad", InCluster: true, Domain: "x.example", + HostAliases: []ClusterHostAlias{{IP: "nope", Hostnames: []string{"a"}}}}, + {ID: "good", InCluster: true, Domain: "x.example", + HostAliases: []ClusterHostAlias{{IP: "10.1.2.3", Hostnames: []string{"A.Example"}}}}, + }, logger) + bad, ok := got["bad"] + if !ok { + t.Fatal("cluster with invalid host_aliases must still be admitted") + } + if bad.HostAliases != nil { + t.Fatalf("invalid host_aliases must be dropped, got %+v", bad.HostAliases) + } + if g := got["good"].HostAliases; len(g) != 1 || g[0].Hostnames[0] != "a.example" { + t.Fatalf("valid host_aliases must be normalized in registry, got %+v", g) + } +} + +func TestHostAliasesConvergedAndMerge(t *testing.T) { + want := []ClusterHostAlias{{IP: "129.34.104.21", Hostnames: []string{"github.ibm.com", "api.github.ibm.com"}}} + + if hostAliasesConverged(nil, want) { + t.Error("empty live must not be converged") + } + live := []byte(`[{"ip":"129.34.104.21","hostnames":["API.github.ibm.com","github.ibm.com"]},{"ip":"10.0.0.9","hostnames":["other.example"]}]`) + if !hostAliasesConverged(live, want) { + t.Error("live superset (any order/case) must be converged") + } + partial := []byte(`[{"ip":"129.34.104.21","hostnames":["github.ibm.com"]}]`) + if hostAliasesConverged(partial, want) { + t.Error("missing hostname must not be converged") + } + if !hostAliasesConverged([]byte(`[]`), nil) { + t.Error("no desired aliases is always converged") + } + + merged := mergeHostAliases([]byte(`[{"ip":"10.0.0.9","hostnames":["other.example"]},{"ip":"129.34.104.21","hostnames":["stale.example"]}]`), want) + if len(merged) != 2 || merged[0].IP != "10.0.0.9" || merged[1].IP != "129.34.104.21" || + strings.Join(merged[1].Hostnames, ",") != "github.ibm.com,api.github.ibm.com" { + t.Fatalf("merge must keep unrelated IPs and replace desired IPs: %+v", merged) + } + + patch, err := hostAliasesPatchJSON(merged) + if err != nil { + t.Fatal(err) + } + var ops []struct { + Op string `json:"op"` + Path string `json:"path"` + Value []ClusterHostAlias `json:"value"` + } + if err := json.Unmarshal([]byte(patch), &ops); err != nil { + t.Fatalf("patch is not valid JSON-Patch: %v\n%s", err, patch) + } + if len(ops) != 1 || ops[0].Op != "add" || ops[0].Path != podHostAliasesPath || len(ops[0].Value) != 2 { + t.Fatalf("unexpected patch: %s", patch) + } +} diff --git a/src/pkg/hub/clusters_registry.go b/src/pkg/hub/clusters_registry.go index e07495bc6e..d8fb409077 100644 --- a/src/pkg/hub/clusters_registry.go +++ b/src/pkg/hub/clusters_registry.go @@ -207,6 +207,16 @@ func validateClusterEntries(configs []ClusterConfig, logger *slog.Logger) map[st logger.Warn("skipping cluster with no domain", "cluster", c.ID) continue } + // host_aliases is validated but NOT an admission rule: a typo there + // must not drop the cluster (and strand its hives); the aliases are + // dropped instead and the operator told why. + aliases, err := normalizeClusterHostAliases(c.HostAliases) + if err != nil { + logger.Warn("ignoring invalid host_aliases for cluster — hives will NOT get them", + "cluster", c.ID, "error", err) + aliases = nil + } + c.HostAliases = aliases clusters[c.ID] = c } return clusters diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index f1652cd756..8ef3b3db00 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -245,6 +245,13 @@ type ClusterConfig struct { // reachable only via the proxy). Injected into new pool slots as // HIVE_UPSTREAM_PROXY_URL / HIVE_UPSTREAM_PROXY_HOSTS / HIVE_NO_PROXY. UpstreamProxy *ClusterUpstreamProxy `json:"upstream_proxy,omitempty" yaml:"upstream_proxy,omitempty"` + // HostAliases are Pod hostAliases entries rendered into every hive + // Deployment on THIS cluster, for GitHub Enterprise hosts reachable only + // through a TLS reverse proxy pinned to a fixed IP (no CONNECT proxy, so + // upstream_proxy cannot express it). Validated at load by + // normalizeClusterHostAliases; existing hives are converged by the + // host_aliases reconcile sweep (cluster_host_aliases.go, #17801). + HostAliases []ClusterHostAlias `json:"host_aliases,omitempty" yaml:"host_aliases,omitempty"` // GitHubAppSlug is the GitHub App's URL slug on THIS cluster's GitHub // host. A GitHub Enterprise instance hosts its own App registration, // which is rarely named "kubestellar-hive" — without this the spoke's @@ -2660,6 +2667,7 @@ func provisionHive(h *SaaSHive, req *CreateHiveRequest, cluster *ClusterConfig, "UpstreamProxyURL": cluster.UpstreamProxy.url(), "UpstreamProxyHosts": cluster.UpstreamProxy.hosts(), "UpstreamProxyNoProxy": cluster.UpstreamProxy.noProxy(), + "HostAliases": cluster.HostAliases, "IsPublic": h.IsPublic, "HiveType": "hosted", "HasGHE": effectiveGitHubBaseURL(h, cluster) != "", @@ -3404,6 +3412,18 @@ spec: app: hive hive-id: {{.ID}} spec: +{{- if .HostAliases}} + # Per-cluster host_aliases (clusters.json): pins GHE hostnames to a TLS + # reverse-proxy IP where no CONNECT proxy exists. See #17801. + hostAliases: +{{- range .HostAliases}} + - ip: "{{.IP}}" + hostnames: +{{- range .Hostnames}} + - "{{.}}" +{{- end}} +{{- end}} +{{- end}} {{- if .RequiresSCC}} serviceAccountName: hive-sa {{- else}} diff --git a/src/pkg/hub/saas_sha_poller.go b/src/pkg/hub/saas_sha_poller.go index 6c04fc0e9f..a7997a6a39 100644 --- a/src/pkg/hub/saas_sha_poller.go +++ b/src/pkg/hub/saas_sha_poller.go @@ -715,6 +715,11 @@ func (s *HubServer) StartLatestSHAPoller(ctx context.Context) { // rate-limited to perHiveEnvMaxPatchesPerCycle patches per cycle, because // each patch rolls that hive's pod. See perhive_env_reconcile.go. s.reconcilePerHiveEnvIfDue() + // Converge per-cluster host_aliases (clusters.json) onto existing hive + // Deployments so GHE-behind-reverse-proxy clusters never need a manual + // kubectl patch again (#17801). Throttled and rate-limited like the lane + // above because each patch rolls a pod. + s.reconcileHostAliasesIfDue() // Put #7457's auth-url / auth-response-headers onto the hive-contribute // Ingress of every hosted spoke provisioned before that fix, so // /api/contribute/me learns who is calling there too (#7517). The @@ -789,6 +794,7 @@ func (s *HubServer) pollLatestSHAsTick(ctx context.Context, now time.Time) { s.reconcileNetAdminIfDue() s.reconcileNodeHealthRBACIfDue() s.reconcilePerHiveEnvIfDue() + s.reconcileHostAliasesIfDue() s.reapOrphanedPodsIfDue() s.retireExpiredGenerationsIfDue() s.sweepExpiredAccessIfDue() diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 07b3e0232b..b024fc42e5 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1210,6 +1210,9 @@ type HubServer struct { // Same rationale and same guarding mutex as lastNetAdminReconcile above: // both are poller-loop-only state. lastPerHiveEnvReconcile time.Time + // lastHostAliasesReconcile throttles the per-cluster host_aliases + // reconcile (cluster_host_aliases.go). Same mutex, same rationale. + lastHostAliasesReconcile time.Time // lastContributeIngressReconcile throttles the hive-contribute Ingress // auth-url reconcile (contribute_ingress_reconcile.go), which puts the // #7457 annotations onto spokes provisioned before that fix (#7517). From 51e8db9b711a97b6d1245054fc2cd9afb439f294 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:24:46 +0000 Subject: [PATCH 033/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20count=20PR=20auth?= =?UTF-8?q?or's=20own=20sign-off=20after=20a=20hive=20rebase;=20no=20decis?= =?UTF-8?q?ion=20gate=20on=20DCO=20asks=20(#17795)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-hive-rebase-signoff-attribution.md | 1 + .../stage_tracker_human_gate_test.go | 7 ++++ src/pkg/dashboard/stage_tracker_rework.go | 15 ++++++++ .../dashboard/stage_tracker_rework_test.go | 8 +++++ .../static-src/js/tracker-human-gate.js | 6 +++- .../dashboard/static-src/js/tracker-words.js | 3 ++ src/pkg/dashboard/static/index.html | 9 ++++- .../github/automerge/automerge_stage_dco.go | 30 +++++++++++++++- .../automerge/automerge_stage_labels_test.go | 36 +++++++++++++++++++ 9 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-hive-rebase-signoff-attribution.md diff --git a/changelog.d/fixed-hive-rebase-signoff-attribution.md b/changelog.d/fixed-hive-rebase-signoff-attribution.md new file mode 100644 index 0000000000..6463d9f05f --- /dev/null +++ b/changelog.d/fixed-hive-rebase-signoff-attribution.md @@ -0,0 +1 @@ +- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. diff --git a/src/pkg/dashboard/stage_tracker_human_gate_test.go b/src/pkg/dashboard/stage_tracker_human_gate_test.go index 53cec8f3a2..e21dc8e0a6 100644 --- a/src/pkg/dashboard/stage_tracker_human_gate_test.go +++ b/src/pkg/dashboard/stage_tracker_human_gate_test.go @@ -202,6 +202,13 @@ assert.deepStrictEqual(ids(['hold'], { kind: 'issue', stage: 10 }), ['Release ho assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row140, { sentinel_notice: true })), []); assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17480, { modifiers: ['needs-decision', 'draft'] })), []); assert.deepStrictEqual(stageTrackerHumanGateReasons({ kind: 'pr', stage: 110, modifiers: ['merged', 'hold'] }), []); +// hivecommons/hive#17789: a 071 row flagged needs_human on a DCO sign-off +// ask is the author's move, not a decision; another 071 reason still is. +const row17780 = { repo: 'hivecommons/hive', number: 17780, kind: 'pr', stage: 71, head: '0e5ccbe', author: 'clubanderson', author_kind: 'human', needs_human: true, reason: 'hive-signoff-attribution' }; +assert.deepStrictEqual(stageTrackerHumanGateReasons(row17780), []); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { reason: 'dco-missing' })), []); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { reason: 'label:needs-rebase' })).map(r => r.key), ['decision']); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { modifiers: ['needs-decision'] })).map(r => r.key), ['decision'], 'an explicit needs-decision still gates'); assert.ok(!stageTrackerRowHtml({ repo: 'acme/console', number: 1, kind: 'pr', stage: 50 }, stages, ctx).includes('ham-human-gate'), 'a clean row has no gate'); // (d) secondary rows #17463 / #17476 of the shared PR #17480 (primary row diff --git a/src/pkg/dashboard/stage_tracker_rework.go b/src/pkg/dashboard/stage_tracker_rework.go index fcd7511ea4..4dbb341738 100644 --- a/src/pkg/dashboard/stage_tracker_rework.go +++ b/src/pkg/dashboard/stage_tracker_rework.go @@ -112,9 +112,24 @@ func stageTrackerReworkOwedBy(it StageTrackerItem) string { if strings.Contains(strings.ToLower(it.Reason), "conflict") && it.Rebase == nil { return reworkOwedByAuthor } + // A DCO ask parks the head on the author's sign-off; no re-review is + // queued for it (hivecommons/hive#17789). + if stageTrackerAuthorDCOReason(it.Reason) { + return reworkOwedByAuthor + } return reworkOwedByHiveReviewer } +// stageTrackerAuthorDCOReason reports a lane reason that asks the author to +// fix their commits' sign-off. +func stageTrackerAuthorDCOReason(reason string) bool { + switch strings.TrimSpace(reason) { + case stagelabels.LaneReasonDCOMissing, stagelabels.LaneReasonHiveSignoffAttribution: + return true + } + return false +} + // StageTrackerFixWork is the hub's fix-queue standing for one PR and one // contributor (ContributeWSHub.FixWorkFor). type StageTrackerFixWork struct { diff --git a/src/pkg/dashboard/stage_tracker_rework_test.go b/src/pkg/dashboard/stage_tracker_rework_test.go index ce47bbd4cb..f6e85b1e1d 100644 --- a/src/pkg/dashboard/stage_tracker_rework_test.go +++ b/src/pkg/dashboard/stage_tracker_rework_test.go @@ -247,6 +247,10 @@ assert.strictEqual(stageTrackerPresenceChipText(working), 'waiting on @Danathar // A pushed head with nothing against it waits on the hive's reviewer (hivecommons/hive#16615). assert.strictEqual(stageTrackerPresenceChipText(pushed), 'waiting on 🐝 hive reviewer · re-review of bbb'); assert.strictEqual(stageTrackerPresenceChipText(w(Object.assign({}, pushed.rework, { owed_by: 'author' }))), 'waiting on @Danathar · re-review pending', 'the server says the author still owes it'); +// hivecommons/hive#17789: an older payload without owed_by reads a DCO ask as the author's move too. +assert.strictEqual(stageTrackerReReviewOwedByHive(Object.assign({}, pushed, { reason: 'hive-signoff-attribution' })), false); +assert.strictEqual(stageTrackerReReviewOwedByHive(Object.assign({}, pushed, { reason: 'dco-missing' })), false); +assert.strictEqual(stageTrackerPresenceChipText(Object.assign({}, pushed, { reason: 'hive-signoff-attribution', rework: Object.assign({}, pushed.rework, { owed_by: 'author' }) })), 'waiting on @Danathar · sign-off owed', 'no re-review is queued for a DCO ask'); assert.match(stageTrackerReworkTip(requested, NOW), /^rework requested 1 hour ago \(\S+ .*\) · no response from @Danathar yet$/); assert.match(stageTrackerReworkTip(queued, NOW), /^rework queued for @Danathar's clanker · 2 ahead · queued 10 min ago \(/); assert.match(stageTrackerReworkTip(acked, NOW), /^@Danathar acknowledged · activity on the PR 20 min ago \(/); @@ -305,6 +309,10 @@ func TestStageTrackerReworkOwedBy16615(t *testing.T) { "red CI on the pushed head": func(it *StageTrackerItem) { it.CIActivity.State = "failing" }, "findings on the pushed head": func(it *StageTrackerItem) { it.ReviewActivity.State = "changes_requested" }, "conflict nobody in the hive rebases": func(it *StageTrackerItem) { it.Reason = "conflicting" }, + // hivecommons/hive#17789: a DCO ask is the author's move; no + // re-review of the pushed head is queued for it. + "hive-signoff-attribution": func(it *StageTrackerItem) { it.Reason = "hive-signoff-attribution" }, + "dco-missing": func(it *StageTrackerItem) { it.Reason = "dco-missing" }, } { it := base() mutate(&it) diff --git a/src/pkg/dashboard/static-src/js/tracker-human-gate.js b/src/pkg/dashboard/static-src/js/tracker-human-gate.js index 6bab762a8a..04ba8f70c8 100644 --- a/src/pkg/dashboard/static-src/js/tracker-human-gate.js +++ b/src/pkg/dashboard/static-src/js/tracker-human-gate.js @@ -83,7 +83,11 @@ const issueDecision = typeof STAGE_TRACKER_ISSUE_NEEDS_DECISION === 'number' ? STAGE_TRACKER_ISSUE_NEEDS_DECISION : 45; // A pre-045 issue still wearing 085 is parked on a person all the same. const decisionStage = pr ? stage === prDecision : (stage === issueDecision || stage === prDecision); - const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || item.needs_human === true; + // A DCO sign-off ask (dco-missing, hive-signoff-attribution) is the + // author's move, named by the waiting-on pill; needs_human alone does + // not make it a decision (hivecommons/hive#17789). + const authorDCO = /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim()); + const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || (item.needs_human === true && !authorDCO); // An unauthorized step and a structured blocker name the person owed // in their own pills (hive-bot-lacks-permission, Blocked · …); they // are not a decision anyone is asked for. diff --git a/src/pkg/dashboard/static-src/js/tracker-words.js b/src/pkg/dashboard/static-src/js/tracker-words.js index b0e95b991e..df5063fcd7 100644 --- a/src/pkg/dashboard/static-src/js/tracker-words.js +++ b/src/pkg/dashboard/static-src/js/tracker-words.js @@ -343,6 +343,8 @@ const r = stageTrackerReworkOf(item); if (!r) return ''; if (r.state === 'requested' && r.evidence === 'queued') return 'queued'; + // A DCO ask on the pushed head queues no re-review (hivecommons/hive#17789). + if (r.state === 'pushed' && /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return 'sign-off owed'; return STAGE_TRACKER_REWORK_WORDS[r.state]; } // "re-review pending" means the HIVE's reviewer owes a pass on the new @@ -360,6 +362,7 @@ const on = k => { const a = item[k + '_activity']; return a && (!a.head || !item.head || a.head === item.head) ? a.state || 'unknown' : 'unknown'; }; if (on('ci') === 'failing' || on('review') === 'changes_requested') return false; if (/conflict/i.test(String(item.reason || '')) && !item.rebase) return false; + if (/^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return false; return true; } // "re-review of d88928e" — the owed pass, named by head. diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 68c2d1d5ce..bba58a4f63 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -18060,6 +18060,8 @@

Where to get help

const r = stageTrackerReworkOf(item); if (!r) return ''; if (r.state === 'requested' && r.evidence === 'queued') return 'queued'; + // A DCO ask on the pushed head queues no re-review (hivecommons/hive#17789). + if (r.state === 'pushed' && /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return 'sign-off owed'; return STAGE_TRACKER_REWORK_WORDS[r.state]; } // "re-review pending" means the HIVE's reviewer owes a pass on the new @@ -18077,6 +18079,7 @@

Where to get help

const on = k => { const a = item[k + '_activity']; return a && (!a.head || !item.head || a.head === item.head) ? a.state || 'unknown' : 'unknown'; }; if (on('ci') === 'failing' || on('review') === 'changes_requested') return false; if (/conflict/i.test(String(item.reason || '')) && !item.rebase) return false; + if (/^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return false; return true; } // "re-review of d88928e" — the owed pass, named by head. @@ -26787,7 +26790,11 @@

All labels

const issueDecision = typeof STAGE_TRACKER_ISSUE_NEEDS_DECISION === 'number' ? STAGE_TRACKER_ISSUE_NEEDS_DECISION : 45; // A pre-045 issue still wearing 085 is parked on a person all the same. const decisionStage = pr ? stage === prDecision : (stage === issueDecision || stage === prDecision); - const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || item.needs_human === true; + // A DCO sign-off ask (dco-missing, hive-signoff-attribution) is the + // author's move, named by the waiting-on pill; needs_human alone does + // not make it a decision (hivecommons/hive#17789). + const authorDCO = /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim()); + const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || (item.needs_human === true && !authorDCO); // An unauthorized step and a structured blocker name the person owed // in their own pills (hive-bot-lacks-permission, Blocked · …); they // are not a decision anyone is asked for. diff --git a/src/pkg/github/automerge/automerge_stage_dco.go b/src/pkg/github/automerge/automerge_stage_dco.go index 5237934be7..a8af6c9969 100644 --- a/src/pkg/github/automerge/automerge_stage_dco.go +++ b/src/pkg/github/automerge/automerge_stage_dco.go @@ -84,7 +84,7 @@ func dcoCommitStatsForHive(commits []*gh.RepositoryCommit, appLogin, prAuthor st } else { stats.unsignedOther++ } - } else if !selfSquash && dcoCommitOwnedByHive(rc, appLogin, isHive) && !messageHasPRAuthorSignoff(msg, prAuthor) { + } else if !selfSquash && dcoCommitOwnedByHive(rc, appLogin, isHive) && !commitHasPRAuthorSignoff(rc, prAuthor) { stats.squashUnsafeHive++ } } @@ -191,6 +191,34 @@ func botSquashAuthor(login string) bool { return strings.HasSuffix(strings.ToLower(strings.TrimSpace(login)), "[bot]") } +// commitHasPRAuthorSignoff mirrors signoff_is_pr_author in +// check-squash-signoff-attribution.sh: a sign-off certifies the PR author +// when it is that account's GitHub noreply, or when GitHub links the commit +// AUTHOR to the PR author and the sign-off is that author's own git email +// (or noreply). The committer is not consulted, so a hive rebase that makes +// the App the committer of the author's own signed commit does not file it +// as the hive's to re-sign (hivecommons/hive#17789). +func commitHasPRAuthorSignoff(commit *gh.RepositoryCommit, prAuthor string) bool { + msg := commit.GetCommit().GetMessage() + if messageHasPRAuthorSignoff(msg, prAuthor) { + return true + } + want := normalizeBotLogin(prAuthor) + if want == "" || normalizeBotLogin(commit.GetAuthor().GetLogin()) != want { + return false + } + authorEmail := strings.ToLower(strings.TrimSpace(commit.GetCommit().GetAuthor().GetEmail())) + for _, line := range strings.Split(msg, "\n") { + if !strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "signed-off-by:") { + continue + } + if email := signedOffEmail(line); email != "" && email == authorEmail { + return true + } + } + return false +} + func messageHasPRAuthorSignoff(message, prAuthor string) bool { login := strings.ToLower(strings.TrimSpace(prAuthor)) if login == "" { diff --git a/src/pkg/github/automerge/automerge_stage_labels_test.go b/src/pkg/github/automerge/automerge_stage_labels_test.go index 01a73647ff..3ed0e74cbb 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_test.go +++ b/src/pkg/github/automerge/automerge_stage_labels_test.go @@ -2465,6 +2465,42 @@ func TestDCOCommitStatsForHiveSquashUnsafeCountsHiveIdentities(t *testing.T) { } } +// hivecommons/hive#17789: a hive rebase + force-push makes the App bot the +// committer of the PR author's own commit. A sign-off with the commit +// author's own email, on a commit GitHub links to the PR author, certifies +// the PR author (the squash attribution gate agrees), so it is never +// squash-unsafe; a foreign sign-off under the same committer still is. +func TestDCOCommitStatsForHiveAuthorSignedAfterHiveRebase(t *testing.T) { + rebased := func(authorLogin, authorEmail, signoff string) *gh.RepositoryCommit { + return &gh.RepositoryCommit{ + Author: &gh.User{Login: gh.Ptr(authorLogin)}, + Committer: &gh.User{Login: gh.Ptr(testHiveAppBotLogin)}, + Commit: &gh.Commit{ + Author: &gh.CommitAuthor{Email: gh.Ptr(authorEmail)}, + Message: gh.Ptr("fix: openshell sandbox names\n\nSigned-off-by: " + signoff), + }, + Parents: []*gh.Commit{{}}, + } + } + for _, tc := range []struct { + name string + commit *gh.RepositoryCommit + want int + }{ + {"author's real email", rebased("clubanderson", "andy@clubanderson.com", "Andrew Anderson "), 0}, + {"author email case differs", rebased("ClubAnderson", "Andy@ClubAnderson.com", "Andrew Anderson "), 0}, + {"author's noreply", rebased("clubanderson", "andy@clubanderson.com", "A <123+clubanderson@users.noreply.github.com>"), 0}, + {"someone else's sign-off", rebased("clubanderson", "andy@clubanderson.com", "Other "), 1}, + {"commit authored by someone else", rebased("collaborator", "andy@clubanderson.com", "Andrew Anderson "), 1}, + {"author login unlinked", rebased("", "andy@clubanderson.com", "Andrew Anderson "), 1}, + } { + stats := dcoCommitStatsForHive([]*gh.RepositoryCommit{tc.commit}, testHiveAppBotLogin, "clubanderson", false, nil) + if stats.unsigned != 0 || stats.squashUnsafeHive != tc.want { + t.Errorf("%s: %+v, want squashUnsafeHive=%d", tc.name, stats, tc.want) + } + } +} + // hivecommons/hive#16701: a split cached before hive identities counted filed // a project.ai_author commit as "other"; stageDCO reads that head's commits // once more so the hive's own commit can be re-signed, then trusts the From ffb68859da69210008a2a5a894a9efdfae71a505 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:26:06 +0000 Subject: [PATCH 034/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20render=20Subst?= =?UTF-8?q?rate=20preflight=20warnings=20on=20the=20agent=20card=20(#17796?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feat: render Substrate preflight warnings on the agent card Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 test: seed proc.Config so sandbox-warnings status test matches the builder's input Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...added-substrate-preflight-card-warnings.md | 1 + src/cmd/hive/main.go | 4 +- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++---- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/config/agent_sandbox_report.go | 45 ++++++++++++++ src/pkg/config/agent_sandbox_report_test.go | 27 +++++++++ src/pkg/config/config.go | 19 ++++-- .../dashboard/agent_sandbox_warnings.test.js | 29 ++++++++++ .../dashboard/agent_sandbox_warnings_test.go | 58 +++++++++++++++++++ src/pkg/dashboard/server.go | 16 +++-- .../dashboard/static-src/js/agent-cards.js | 7 +++ src/pkg/dashboard/static/index.html | 7 +++ src/pkg/dashboard/status_builder.go | 1 + 15 files changed, 218 insertions(+), 28 deletions(-) create mode 100644 changelog.d/added-substrate-preflight-card-warnings.md create mode 100644 src/pkg/config/agent_sandbox_report.go create mode 100644 src/pkg/config/agent_sandbox_report_test.go create mode 100644 src/pkg/dashboard/agent_sandbox_warnings.test.js create mode 100644 src/pkg/dashboard/agent_sandbox_warnings_test.go diff --git a/changelog.d/added-substrate-preflight-card-warnings.md b/changelog.d/added-substrate-preflight-card-warnings.md new file mode 100644 index 0000000000..e3bbc51847 --- /dev/null +++ b/changelog.d/added-substrate-preflight-card-warnings.md @@ -0,0 +1 @@ +- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 0c660bc8cb..bd17e6e5b7 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -7144,7 +7144,9 @@ func parseColorInt(color string) int { // operator who flips the Security tab's sandbox toggle never restarts, so a // boot-only check would never reach the person who most needs it. func logAgentSandboxPosture(logger *slog.Logger, cfg *config.Config) { - for _, warning := range config.AgentSandboxGateWarnings(cfg) { + report := config.AgentSandboxGateReport(cfg) + config.StoreSandboxGateReport(report) + for _, warning := range report.Warnings { logger.Warn("agent sandbox posture", "warning", warning) } } diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 08dba0af9e..8c5b76d503 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1683` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1684` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1685` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1687` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1688` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1689` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index c8255286a9..aec304411e 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1690` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1694` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index e41ee91d74..a0737fea7e 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1663` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1664` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1665` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1671` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1676` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1677` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1680` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1682` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1667` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1668` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1669` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1675` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1680` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1682` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1683` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1684` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1685` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1686` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index b8e71aaf96..96caa5bfa3 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1753` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1757` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/config/agent_sandbox_report.go b/src/pkg/config/agent_sandbox_report.go new file mode 100644 index 0000000000..b21a42c1b0 --- /dev/null +++ b/src/pkg/config/agent_sandbox_report.go @@ -0,0 +1,45 @@ +package config + +import "sync" + +// SandboxGateReport is the result of AgentSandboxGateReport. OpenShell and +// Substrate are the preflight findings for agents on that runtime; both are +// also included in Warnings. +type SandboxGateReport struct { + Warnings []string + OpenShell []string + Substrate []string +} + +var ( + sandboxGateReportMu sync.RWMutex + sandboxGateReport SandboxGateReport +) + +// StoreSandboxGateReport records the report computed at boot/reload so +// readers such as the status builder never run a preflight themselves. +func StoreSandboxGateReport(r SandboxGateReport) { + sandboxGateReportMu.Lock() + defer sandboxGateReportMu.Unlock() + sandboxGateReport = r +} + +// LastSandboxGateReport returns the report last stored by StoreSandboxGateReport. +func LastSandboxGateReport() SandboxGateReport { + sandboxGateReportMu.RLock() + defer sandboxGateReportMu.RUnlock() + return sandboxGateReport +} + +// SandboxPreflightWarningsFor returns the stored preflight warnings that apply +// to an agent on the given sandbox runtime. +func SandboxPreflightWarningsFor(runtime string) []string { + r := LastSandboxGateReport() + switch runtime { + case SandboxRuntimeOpenShell: + return r.OpenShell + case SandboxRuntimeSubstrate: + return r.Substrate + } + return nil +} diff --git a/src/pkg/config/agent_sandbox_report_test.go b/src/pkg/config/agent_sandbox_report_test.go new file mode 100644 index 0000000000..f1e0115515 --- /dev/null +++ b/src/pkg/config/agent_sandbox_report_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxPreflightWarningsFor(t *testing.T) { + t.Cleanup(func() { StoreSandboxGateReport(SandboxGateReport{}) }) + StoreSandboxGateReport(SandboxGateReport{ + OpenShell: []string{"o"}, + Substrate: []string{"s"}, + }) + for _, tc := range []struct { + runtime string + want []string + }{ + {SandboxRuntimeOpenShell, []string{"o"}}, + {SandboxRuntimeSubstrate, []string{"s"}}, + {SandboxRuntimeJob, nil}, + {"", nil}, + } { + if got := SandboxPreflightWarningsFor(tc.runtime); !reflect.DeepEqual(got, tc.want) { + t.Errorf("SandboxPreflightWarningsFor(%q) = %v, want %v", tc.runtime, got, tc.want) + } + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 92917c4108..6c28e87550 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1586,10 +1586,17 @@ func SplitCadenceTargetKey(key string) (agent, repo string) { // legal config, and refusing to boot on it would turn a diagnostic into an // outage. func AgentSandboxGateWarnings(cfg *Config) []string { + return AgentSandboxGateReport(cfg).Warnings +} + +// AgentSandboxGateReport is AgentSandboxGateWarnings plus the preflight +// findings kept apart per runtime, so callers can attribute them to the agents +// on that runtime without parsing strings. Warnings is the same flat list. +func AgentSandboxGateReport(cfg *Config) SandboxGateReport { if cfg == nil || !cfg.AgentSandbox.Enabled { // The sandbox is off globally. That is the documented default and the // posture the docs describe; it is not a misconfiguration. - return nil + return SandboxGateReport{} } var optedIn, noImage, noClaim, noGateway, noDigest, badRuntime []string @@ -1642,7 +1649,7 @@ func AgentSandboxGateWarnings(cfg *Config) []string { "agent_sandbox.enabled is true but NO agent is opted in, so the sandbox is inert and all %d agent(s) still run unconfined on the tmux path — "+ "the per-agent gate is separate: set `sandbox: {enabled: true}` on each agent that should be sandboxed (#4918)", len(cfg.Agents))) - return out + return SandboxGateReport{Warnings: out} } if len(noImage) > 0 { out = append(out, fmt.Sprintf( @@ -1681,9 +1688,11 @@ func AgentSandboxGateWarnings(cfg *Config) []string { } // Last, because it may reach the network (bounded and cached): the static // warnings above must not wait on a gateway. - out = append(out, cruxconfig.OpenShellPreflightWarnings(openshellBlocks)...) - out = append(out, cruxconfig.SubstratePreflightWarnings(substrateBlocks)...) - return out + openshellWarnings := cruxconfig.OpenShellPreflightWarnings(openshellBlocks) + substrateWarnings := cruxconfig.SubstratePreflightWarnings(substrateBlocks) + out = append(out, openshellWarnings...) + out = append(out, substrateWarnings...) + return SandboxGateReport{Warnings: out, OpenShell: openshellWarnings, Substrate: substrateWarnings} } // CadenceIsOperatorOwned reports whether an operator explicitly set the diff --git a/src/pkg/dashboard/agent_sandbox_warnings.test.js b/src/pkg/dashboard/agent_sandbox_warnings.test.js new file mode 100644 index 0000000000..4c5b8dc18d --- /dev/null +++ b/src/pkg/dashboard/agent_sandbox_warnings.test.js @@ -0,0 +1,29 @@ +// Agent card renders the per-agent sandbox_warnings preflight findings. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const src = fs.readFileSync(path.join(__dirname, 'static-src', 'js', 'agent-cards.js'), 'utf8'); +const m = src.match(/function agentSandboxWarningsHtml\(a\) \{[\s\S]*?\n \}\n/); +assert.ok(m, 'agentSandboxWarningsHtml not found'); +const esc = s => String(s).replace(/&/g, '&').replace(/ { + const html = agentSandboxWarningsHtml({ sandbox_warnings: ['operator missing', 'no '] }); + assert.strictEqual((html.match(/class="agent-error"/g) || []).length, 2); + assert.ok(html.includes('⚠ operator missing')); + assert.ok(html.includes('no <gvisor>')); +}); + +test('renders nothing without warnings', () => { + assert.strictEqual(agentSandboxWarningsHtml({}), ''); + assert.strictEqual(agentSandboxWarningsHtml({ sandbox_warnings: [] }), ''); + assert.strictEqual(agentSandboxWarningsHtml(null), ''); +}); + +test('shipped index.html carries the card hook', () => { + const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); + assert.ok(html.includes('${agentSandboxWarningsHtml(a)}')); +}); diff --git a/src/pkg/dashboard/agent_sandbox_warnings_test.go b/src/pkg/dashboard/agent_sandbox_warnings_test.go new file mode 100644 index 0000000000..2bb774ca7e --- /dev/null +++ b/src/pkg/dashboard/agent_sandbox_warnings_test.go @@ -0,0 +1,58 @@ +package dashboard + +import ( + "reflect" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +func TestAgentStatusSandboxWarningsByRuntime(t *testing.T) { + config.StoreSandboxGateReport(config.SandboxGateReport{ + OpenShell: []string{"openshell gateway unreachable"}, + Substrate: []string{"substrate operator not found"}, + }) + t.Cleanup(func() { config.StoreSandboxGateReport(config.SandboxGateReport{}) }) + + enabled := true + agent := func(runtime string, sandboxed bool) config.AgentConfig { + a := config.AgentConfig{Backend: "claude", Enabled: true} + if sandboxed { + a.Sandbox = &config.AgentSandboxOverride{Enabled: &enabled, Runtime: runtime} + } + return a + } + cfg := &config.Config{ + AgentRuntimeConfig: config.AgentRuntimeConfig{ + AgentSandbox: config.AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]config.AgentConfig{ + "sub": agent("substrate", true), + "osh": agent("openshell", true), + "job": agent("job", true), + "none": agent("", false), + }, + }, + Governor: config.GovernorConfig{Modes: map[string]config.ModeConfig{}}, + } + statuses := map[string]*crux.Process{} + for n, a := range cfg.Agents { + statuses[n] = &crux.Process{Config: a} + } + payload := BuildAgentOnlyStatus(governor.State{Mode: "IDLE"}, statuses, cfg) + + want := map[string][]string{ + "sub": {"substrate operator not found"}, + "osh": {"openshell gateway unreachable"}, + "job": nil, + "none": nil, + } + got := map[string][]string{} + for _, a := range payload.Agents { + got[a.Name] = a.SandboxWarnings + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("sandbox warnings by agent = %#v, want %#v", got, want) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 7a72a42eb2..e1db31882f 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -955,12 +955,16 @@ type FrontendAgent struct { // SandboxRuntime is the effective sandbox.runtime for a sandboxed agent // ("podman" or "job"); the SPA routes "▶ terminal" to the Job pod log // viewer for "job" (hivecommons/hive#14798). Empty when not sandboxed. - SandboxRuntime string `json:"sandboxRuntime,omitempty"` - LastError string `json:"lastError,omitempty"` - StallNudges int `json:"stallNudges,omitempty"` - StallRecoveries int `json:"stallRecoveries,omitempty"` - ActionNudges int `json:"actionNudges,omitempty"` - TransientNudges int `json:"transientNudges,omitempty"` + SandboxRuntime string `json:"sandboxRuntime,omitempty"` + // SandboxWarnings are the boot/reload preflight findings for this agent's + // sandbox runtime (openshell or substrate), from the stored + // config.AgentSandboxGateReport; no preflight runs on the status path. + SandboxWarnings []string `json:"sandbox_warnings,omitempty"` + LastError string `json:"lastError,omitempty"` + StallNudges int `json:"stallNudges,omitempty"` + StallRecoveries int `json:"stallRecoveries,omitempty"` + ActionNudges int `json:"actionNudges,omitempty"` + TransientNudges int `json:"transientNudges,omitempty"` // SessionRssMb is the resident memory of the agent's CLI process tree // (MiB) and SessionAge how long the current CLI session has been up, // both sampled on the health tick (#14842). SessionRecycles counts diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index 1fdf34d621..db730d57dd 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -747,6 +747,12 @@ tmux session or in-pod process to be "down". Their health comes from the server-side sandboxRun verdict (last run + cadence), never from state !== 'running'. */ + /* Preflight findings for this agent's sandbox runtime (openshell or + substrate), computed at boot/reload and carried as sandbox_warnings. */ + function agentSandboxWarningsHtml(a) { + const ws = a && Array.isArray(a.sandbox_warnings) ? a.sandbox_warnings : []; + return ws.map(w => `
⚠ ${esc(w)}
`).join(''); + } function agentSandboxRun(a) { return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; } @@ -985,6 +991,7 @@ const prefix = retrying ? 'retrying: ' : ''; return `
⚠ ${esc(prefix)}${esc(a.lastError)}
`; })()} + ${agentSandboxWarningsHtml(a)}
${isInferenceBackendName(a.cli) ? 'method' : 'cli'}${cliChip(a.cli, a.pinnedBoth || a.pinnedCli, a.name)}${(a.pinnedBoth || a.pinnedCli) ? '' : ` `}
model${modelChip(a.model, a.govReason, a.pinnedBoth || a.pinnedModel, a.name)}${(a.pinnedBoth || a.pinnedModel) ? '' : ` `}
diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index bba58a4f63..fd73ff80e2 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -30945,6 +30945,12 @@

All labels

tmux session or in-pod process to be "down". Their health comes from the server-side sandboxRun verdict (last run + cadence), never from state !== 'running'. */ + /* Preflight findings for this agent's sandbox runtime (openshell or + substrate), computed at boot/reload and carried as sandbox_warnings. */ + function agentSandboxWarningsHtml(a) { + const ws = a && Array.isArray(a.sandbox_warnings) ? a.sandbox_warnings : []; + return ws.map(w => `
⚠ ${esc(w)}
`).join(''); + } function agentSandboxRun(a) { return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; } @@ -31183,6 +31189,7 @@

All labels

const prefix = retrying ? 'retrying: ' : ''; return `
⚠ ${esc(prefix)}${esc(a.lastError)}
`; })()} + ${agentSandboxWarningsHtml(a)}
${isInferenceBackendName(a.cli) ? 'method' : 'cli'}${cliChip(a.cli, a.pinnedBoth || a.pinnedCli, a.name)}${(a.pinnedBoth || a.pinnedCli) ? '' : ` `}
model${modelChip(a.model, a.govReason, a.pinnedBoth || a.pinnedModel, a.name)}${(a.pinnedBoth || a.pinnedModel) ? '' : ` `}
diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index ff2e6c23c2..e85b6beac4 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1156,6 +1156,7 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config Sandboxed: agentCfg.SandboxEnabled(cfg.AgentSandbox), SandboxRun: sandboxRun, SandboxRuntime: effectiveSandboxRuntime(agentCfg, cfg), + SandboxWarnings: config.SandboxPreflightWarningsFor(effectiveSandboxRuntime(agentCfg, cfg)), Session: name, State: string(proc.State), Starting: proc.Starting && proc.State != crux.StateRunning, From 0917022ba565a667639c645e36d0a3bc126c3c6c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:43:36 +0000 Subject: [PATCH 035/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20dedupe=20repeated?= =?UTF-8?q?=20undelivered=20review=20kick=20warning=20(#17755)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-review-kick-undelivered-dedupe.md | 1 + src/cmd/hive/review_kick_order.go | 46 +++++++++++++++++++ src/cmd/hive/review_kick_order_test.go | 22 +++++++++ 3 files changed, 69 insertions(+) create mode 100644 changelog.d/fixed-review-kick-undelivered-dedupe.md diff --git a/changelog.d/fixed-review-kick-undelivered-dedupe.md b/changelog.d/fixed-review-kick-undelivered-dedupe.md new file mode 100644 index 0000000000..974cf9e77e --- /dev/null +++ b/changelog.d/fixed-review-kick-undelivered-dedupe.md @@ -0,0 +1 @@ +- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. diff --git a/src/cmd/hive/review_kick_order.go b/src/cmd/hive/review_kick_order.go index ba5aa40aa0..a949df169b 100644 --- a/src/cmd/hive/review_kick_order.go +++ b/src/cmd/hive/review_kick_order.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "sync" + "time" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/scheduler" @@ -99,10 +100,20 @@ func orderReviewFixKicksFirst(messages []scheduler.KickMessage, reviewKickByMess // particular PR is being starved. The planner drops the undelivered kick // from the dispatch state (review.ConfirmDelivered) and re-plans it on the // next cycle, so the line says so. +// +// The same kick for the same PR head fails with the same cause every cycle +// while an agent's pane is not ready, so repeats are logged at debug (and +// not re-audited) until the cause changes or undeliveredKickRewarnAfter +// passes. func logUndeliveredReviewKick(logger *slog.Logger, auditor reviewFixAuditor, k review.DispatchKick, err error) { if logger == nil { logger = slog.Default() } + if !undeliveredKicks.shouldReport(k, err, time.Now()) { + logger.Debug("review swarm kick still not delivered", + "kind", k.Kind, "agent", k.Agent, "pr", k.PRRef, "head", k.HeadSHA, "error", err) + return + } logger.Warn("review swarm kick not delivered; re-planned next cycle", "kind", k.Kind, "agent", k.Agent, "pr", k.PRRef, "head", k.HeadSHA, "error", err) if auditor == nil { @@ -112,6 +123,41 @@ func logUndeliveredReviewKick(logger *slog.Logger, auditor reviewFixAuditor, k r fmt.Sprintf("kind=%s pr=%s head=%s agent=%s error=%v", k.Kind, k.PRRef, k.HeadSHA, k.Agent, err), k.Agent) } +const ( + undeliveredKickRewarnAfter = 30 * time.Minute + undeliveredKickMaxTracked = 1024 +) + +type undeliveredKickReport struct { + cause string + at time.Time +} + +type undeliveredKickTracker struct { + mu sync.Mutex + seen map[string]undeliveredKickReport +} + +var undeliveredKicks = &undeliveredKickTracker{seen: map[string]undeliveredKickReport{}} + +func (t *undeliveredKickTracker) shouldReport(k review.DispatchKick, err error, now time.Time) bool { + key := k.Kind + "\x00" + k.Agent + "\x00" + k.PRRef + "\x00" + k.HeadSHA + cause := "" + if err != nil { + cause = err.Error() + } + t.mu.Lock() + defer t.mu.Unlock() + if prev, ok := t.seen[key]; ok && prev.cause == cause && now.Sub(prev.at) < undeliveredKickRewarnAfter { + return false + } + if len(t.seen) >= undeliveredKickMaxTracked { + t.seen = map[string]undeliveredKickReport{} + } + t.seen[key] = undeliveredKickReport{cause: cause, at: now} + return true +} + // governorKickDisplacementLimit is how many consecutive cycles an agent's // governor kick may be deferred behind a delivered fix kick before the // governor kick goes first for one cycle. Two: a single displacement is the diff --git a/src/cmd/hive/review_kick_order_test.go b/src/cmd/hive/review_kick_order_test.go index 6822ee190b..8a45547f40 100644 --- a/src/cmd/hive/review_kick_order_test.go +++ b/src/cmd/hive/review_kick_order_test.go @@ -87,6 +87,28 @@ func TestLogUndeliveredReviewKickNamesPR(t *testing.T) { logUndeliveredReviewKick(logger, nil, k, errors.New("x")) } +// The same undelivered kick (same PR head, same cause) repeats every cycle +// while the pane is not ready; only the first occurrence may warn/audit. +func TestLogUndeliveredReviewKickDedupesRepeats(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + auditor := &recordingFixAuditor{} + k := review.DispatchKick{Kind: "fix", Agent: "dedupe-agent", PRRef: "hivecommons/hive#17724", HeadSHA: "abc1234"} + for i := 0; i < 5; i++ { + logUndeliveredReviewKick(logger, auditor, k, errors.New("pane not ready")) + } + if n := strings.Count(buf.String(), "review swarm kick not delivered"); n != 1 { + t.Fatalf("warn count = %d, want 1: %s", n, buf.String()) + } + if len(auditor.entries) != 1 { + t.Fatalf("audit entries = %d, want 1", len(auditor.entries)) + } + logUndeliveredReviewKick(logger, auditor, k, errors.New("different cause")) + if n := strings.Count(buf.String(), "review swarm kick not delivered"); n != 2 { + t.Fatalf("warn count after cause change = %d, want 2", n) + } +} + // hivecommons/hive#15166: a fix kick delivered first leaves the lane busy, so // the governor kick behind it is deferred; an adopted PR under // changes_requested is re-planned as a fix kick every cycle, so the scanner's From fd0ec4427b424071e7067509cc5afd6c11d10225 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:43:56 +0000 Subject: [PATCH 036/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20cache=20app=5Fkey?= =?UTF-8?q?s=5Fheld=20key=20fingerprints=20so=20heartbeat=20collect=20stay?= =?UTF-8?q?s=20in=20budget=20(#17806)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-app-keys-held-fingerprint-cache.md | 1 + src/pkg/appkey/appkey.go | 37 +++++++++++++++++-- src/pkg/appkey/write_error_test.go | 33 +++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-app-keys-held-fingerprint-cache.md diff --git a/changelog.d/fixed-app-keys-held-fingerprint-cache.md b/changelog.d/fixed-app-keys-held-fingerprint-cache.md new file mode 100644 index 0000000000..82e292b784 --- /dev/null +++ b/changelog.d/fixed-app-keys-held-fingerprint-cache.md @@ -0,0 +1 @@ +- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. diff --git a/src/pkg/appkey/appkey.go b/src/pkg/appkey/appkey.go index 1e8772937f..9a57a80fc0 100644 --- a/src/pkg/appkey/appkey.go +++ b/src/pkg/appkey/appkey.go @@ -20,6 +20,8 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "time" "github.com/hivecommons/hive/pkg/config" ) @@ -62,6 +64,17 @@ type Resolver struct { ProvisionedKeyPath string // FileMode is applied to keys this Resolver writes. FileMode os.FileMode + + fpMu sync.Mutex + fpCache map[string]heldFingerprint +} + +// heldFingerprint memoizes a key file's fingerprint against the size and mtime +// it was computed from, so an unchanged file is never re-read. +type heldFingerprint struct { + size int64 + mtime time.Time + fp string } type tempKeyFile interface { @@ -77,8 +90,9 @@ var ( createTempKey = func(dir, pattern string) (tempKeyFile, error) { return os.CreateTemp(dir, pattern) } - removeFile = os.Remove - renameFile = os.Rename + fingerprintFromFile = config.AppKeyFingerprintFromFile + removeFile = os.Remove + renameFile = os.Rename ) // Default returns a Resolver using the production spoke layout. @@ -125,6 +139,10 @@ func (r *Resolver) HeldFingerprints() map[string]string { return nil } var held map[string]string + r.fpMu.Lock() + defer r.fpMu.Unlock() + seen := make(map[string]heldFingerprint) + defer func() { r.fpCache = seen }() for _, e := range entries { if e.IsDir() { continue @@ -138,10 +156,21 @@ func (r *Resolver) HeldFingerprints() map[string]string { if convErr != nil || id <= 0 { continue } - fp, fpErr := config.AppKeyFingerprintFromFile(filepath.Join(r.DataDir, name)) - if fpErr != nil || fp == "" { + var fp string + info, infoErr := e.Info() + if infoErr != nil { continue } + if c, ok := r.fpCache[name]; ok && c.size == info.Size() && c.mtime.Equal(info.ModTime()) { + fp = c.fp + } else { + var fpErr error + fp, fpErr = fingerprintFromFile(filepath.Join(r.DataDir, name)) + if fpErr != nil || fp == "" { + continue + } + } + seen[name] = heldFingerprint{size: info.Size(), mtime: info.ModTime(), fp: fp} if held == nil { held = make(map[string]string) } diff --git a/src/pkg/appkey/write_error_test.go b/src/pkg/appkey/write_error_test.go index 084a2ac422..0e7c91a1fa 100644 --- a/src/pkg/appkey/write_error_test.go +++ b/src/pkg/appkey/write_error_test.go @@ -338,3 +338,36 @@ func TestWriteKeyFileAtomicReplacesUnwritableFile(t *testing.T) { t.Fatal("rejected writes must leave the key untouched") } } + +func TestHeldFingerprintsDoesNotRereadUnchangedKeys(t *testing.T) { + r := newTestResolver(t) + if err := os.MkdirAll(r.DataDir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(r.DataDir, "gh-app-key-42.pem") + writeTestAppKey(t, path) + + reads := 0 + orig := fingerprintFromFile + fingerprintFromFile = func(p string) (string, error) { + reads++ + return orig(p) + } + t.Cleanup(func() { fingerprintFromFile = orig }) + + first := r.HeldFingerprints() + second := r.HeldFingerprints() + if first["42"] == "" || second["42"] != first["42"] { + t.Fatalf("fingerprints = %v then %v", first, second) + } + if reads != 1 { + t.Fatalf("key file read %d times across two scans, want 1", reads) + } + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if got := r.HeldFingerprints(); got != nil { + t.Fatalf("removed key still reported: %v", got) + } +} From 34cbb4fd5b1ec0339abe2b5f47a7b79883479bd5 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 00:55:22 +0000 Subject: [PATCH 037/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.225.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.225.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 20 +++++++++++++++++++ changelog.d/added-hub-cluster-host-aliases.md | 1 - .../added-settings-substrate-leaves.md | 1 - .../fixed-agent-lifecycle-repeated-class.md | 1 - .../fixed-automerge-orphaned-queued-runs.md | 1 - .../fixed-claude-shared-credentials.md | 1 - .../fixed-hive-rebase-signoff-attribution.md | 1 - .../fixed-hub-digest-stale-tracker-read.md | 1 - .../fixed-issue-claims-retry-warn-dedupe.md | 1 - .../security-marketplace-model-provenance.md | 1 - 10 files changed, 20 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/added-hub-cluster-host-aliases.md delete mode 100644 changelog.d/added-settings-substrate-leaves.md delete mode 100644 changelog.d/fixed-agent-lifecycle-repeated-class.md delete mode 100644 changelog.d/fixed-automerge-orphaned-queued-runs.md delete mode 100644 changelog.d/fixed-claude-shared-credentials.md delete mode 100644 changelog.d/fixed-hive-rebase-signoff-attribution.md delete mode 100644 changelog.d/fixed-hub-digest-stale-tracker-read.md delete mode 100644 changelog.d/fixed-issue-claims-retry-warn-dedupe.md delete mode 100644 changelog.d/security-marketplace-model-provenance.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e157c27e3..8cb83973d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,26 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.225.0) + +### Added + +- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). +- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. + +### Fixed + +- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. +- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). +- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. +- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. +- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. +- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). + +### Security + +- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. + ## 2026-11-12 (v5.224.0) ### Added diff --git a/changelog.d/added-hub-cluster-host-aliases.md b/changelog.d/added-hub-cluster-host-aliases.md deleted file mode 100644 index ebc6e54a5c..0000000000 --- a/changelog.d/added-hub-cluster-host-aliases.md +++ /dev/null @@ -1 +0,0 @@ -- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). diff --git a/changelog.d/added-settings-substrate-leaves.md b/changelog.d/added-settings-substrate-leaves.md deleted file mode 100644 index 59d543165c..0000000000 --- a/changelog.d/added-settings-substrate-leaves.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. diff --git a/changelog.d/fixed-agent-lifecycle-repeated-class.md b/changelog.d/fixed-agent-lifecycle-repeated-class.md deleted file mode 100644 index fda13f7ad7..0000000000 --- a/changelog.d/fixed-agent-lifecycle-repeated-class.md +++ /dev/null @@ -1 +0,0 @@ -- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. diff --git a/changelog.d/fixed-automerge-orphaned-queued-runs.md b/changelog.d/fixed-automerge-orphaned-queued-runs.md deleted file mode 100644 index 6e7f72b821..0000000000 --- a/changelog.d/fixed-automerge-orphaned-queued-runs.md +++ /dev/null @@ -1 +0,0 @@ -- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). diff --git a/changelog.d/fixed-claude-shared-credentials.md b/changelog.d/fixed-claude-shared-credentials.md deleted file mode 100644 index db3fa495ed..0000000000 --- a/changelog.d/fixed-claude-shared-credentials.md +++ /dev/null @@ -1 +0,0 @@ -- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. diff --git a/changelog.d/fixed-hive-rebase-signoff-attribution.md b/changelog.d/fixed-hive-rebase-signoff-attribution.md deleted file mode 100644 index 6463d9f05f..0000000000 --- a/changelog.d/fixed-hive-rebase-signoff-attribution.md +++ /dev/null @@ -1 +0,0 @@ -- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. diff --git a/changelog.d/fixed-hub-digest-stale-tracker-read.md b/changelog.d/fixed-hub-digest-stale-tracker-read.md deleted file mode 100644 index 12611376c5..0000000000 --- a/changelog.d/fixed-hub-digest-stale-tracker-read.md +++ /dev/null @@ -1 +0,0 @@ -- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. diff --git a/changelog.d/fixed-issue-claims-retry-warn-dedupe.md b/changelog.d/fixed-issue-claims-retry-warn-dedupe.md deleted file mode 100644 index 0942f81c03..0000000000 --- a/changelog.d/fixed-issue-claims-retry-warn-dedupe.md +++ /dev/null @@ -1 +0,0 @@ -- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). diff --git a/changelog.d/security-marketplace-model-provenance.md b/changelog.d/security-marketplace-model-provenance.md deleted file mode 100644 index 2f6054f7a6..0000000000 --- a/changelog.d/security-marketplace-model-provenance.md +++ /dev/null @@ -1 +0,0 @@ -- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. From b111d7b64fb1b16557b8c6e2ff601d3f09736344 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:05:05 +0000 Subject: [PATCH 038/223] =?UTF-8?q?=F0=9F=90=9B=20fix(marketplace):=20rest?= =?UTF-8?q?rict=20spoke-triggered=20=3Frepo=3D=20refresh=20to=20tracked=20?= =?UTF-8?q?repositories=20(#17800)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(marketplace): restrict spoke-triggered ?repo= refresh to tracked repositories A registered spoke could pass any owner/name to POST /api/marketplace/refresh and the hub would fetch it with its own GitHub identity and publish the ingested model under the hub's name. Non-admin callers must now name a corpus or HIVE_HUB_MARKETPLACE_EXTRA_REPOS entry; admins are unchanged. Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: refresh head to re-report required checks Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-marketplace-refresh-corpus-only.md | 1 + src/pkg/hub/marketplace.go | 10 +++++++++- src/pkg/hub/marketplace_refresh.go | 12 ++++++++++++ src/pkg/hub/marketplace_test.go | 17 +++++++++++++++-- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 changelog.d/security-marketplace-refresh-corpus-only.md diff --git a/changelog.d/security-marketplace-refresh-corpus-only.md b/changelog.d/security-marketplace-refresh-corpus-only.md new file mode 100644 index 0000000000..9c888537a7 --- /dev/null +++ b/changelog.d/security-marketplace-refresh-corpus-only.md @@ -0,0 +1 @@ +- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 0388838bb4..c886fcac26 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -222,7 +222,11 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ // catalogue" button works through the spoke. Spokes are rate-limited (one // full run per marketplaceFullRefreshInterval across all spokes, // marketplaceRepoRefreshBurst single-repo runs per minute) → 429 with -// Retry-After; admins are not. +// Retry-After; admins are not. A spoke's ?repo= must also name a repository +// the hub already tracks (the corpus or HIVE_HUB_MARKETPLACE_EXTRA_REPOS), +// otherwise 403: the ingest runs with the hub's GitHub identity and +// publishes under the hub's name, so only an admin may widen what it +// fetches. func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return @@ -241,6 +245,10 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ return } if !who.admin { + if !marketplaceTracksRepo(repo) { + marketplaceJSONError(w, http.StatusForbidden, "repo is not in the marketplace corpus; only a hub admin may refresh other repositories") + return + } if ok, wait := rf.allowRepoRefresh(now); !ok { marketplaceTooMany(w, wait, "single-repository refresh") return diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index baf5d208a7..f55dd82fc0 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -192,6 +192,18 @@ func marketplaceRepos() []string { return out } +// marketplaceTracksRepo reports whether repo (already lower-cased and +// trimmed) is one the hub ingests on its own: a corpus entry or an +// HIVE_HUB_MARKETPLACE_EXTRA_REPOS entry. +func marketplaceTracksRepo(repo string) bool { + for _, r := range marketplaceRepos() { + if r == repo { + return true + } + } + return false +} + // marketplaceDisabled reports HIVE_HUB_MARKETPLACE_DISABLED. func marketplaceDisabled() bool { v := strings.ToLower(strings.TrimSpace(os.Getenv(hubMarketplaceDisableEnv))) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index d62f22c3f0..e28ba73458 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -796,7 +796,7 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { s := marketplaceTestHub(t, fetch) clock := s.refresher().now() s.refresher().now = func() time.Time { return clock } - t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo") + t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo,ok/repo") spoke := func(target string) *http.Request { req := httptest.NewRequest(http.MethodPost, target, nil) @@ -831,8 +831,21 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { t.Fatalf("bad bearer: %d", rec.Code) } + // Spoke single repo outside the corpus: refused before any fetch or + // rate-limit accounting; the same request as admin is allowed. + rec := mktDo(s, spoke("/api/marketplace/refresh?repo=outside/corpus")) + if rec.Code != 403 || !strings.Contains(rec.Body.String(), "not in the marketplace corpus") { + t.Fatalf("spoke outside corpus: %d %s", rec.Code, rec.Body.String()) + } + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh?repo=outside/corpus")); rec.Code != 200 { + t.Fatalf("admin outside corpus: %d %s", rec.Code, rec.Body.String()) + } + if _, ok := s.marketplace().refreshRecord("outside/corpus"); !ok { + t.Fatal("admin refresh of an outside-corpus repo should have run") + } + // Spoke single repo: ok, then burst-limited. - rec := mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) + rec = mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) if rec.Code != 200 || !strings.Contains(rec.Body.String(), "404 Not Found") { t.Fatalf("spoke single: %d %s", rec.Code, rec.Body.String()) } From 9be929fc84a4824c94bfc9a0bb3309f0fc4fefbc Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 01:27:27 +0000 Subject: [PATCH 039/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.226.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.226.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 11 +++++++++++ .../added-substrate-preflight-card-warnings.md | 1 - changelog.d/fixed-app-keys-held-fingerprint-cache.md | 1 - changelog.d/fixed-review-kick-undelivered-dedupe.md | 1 - 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/added-substrate-preflight-card-warnings.md delete mode 100644 changelog.d/fixed-app-keys-held-fingerprint-cache.md delete mode 100644 changelog.d/fixed-review-kick-undelivered-dedupe.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb83973d8..4da2380372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.226.0) + +### Added + +- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). + +### Fixed + +- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. +- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. + ## 2026-11-12 (v5.225.0) ### Added diff --git a/changelog.d/added-substrate-preflight-card-warnings.md b/changelog.d/added-substrate-preflight-card-warnings.md deleted file mode 100644 index e3bbc51847..0000000000 --- a/changelog.d/added-substrate-preflight-card-warnings.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). diff --git a/changelog.d/fixed-app-keys-held-fingerprint-cache.md b/changelog.d/fixed-app-keys-held-fingerprint-cache.md deleted file mode 100644 index 82e292b784..0000000000 --- a/changelog.d/fixed-app-keys-held-fingerprint-cache.md +++ /dev/null @@ -1 +0,0 @@ -- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. diff --git a/changelog.d/fixed-review-kick-undelivered-dedupe.md b/changelog.d/fixed-review-kick-undelivered-dedupe.md deleted file mode 100644 index 974cf9e77e..0000000000 --- a/changelog.d/fixed-review-kick-undelivered-dedupe.md +++ /dev/null @@ -1 +0,0 @@ -- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. From e06e1b6c2a66b1e53e633ee775116d24d013b9c5 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:30:22 +0000 Subject: [PATCH 040/223] =?UTF-8?q?=F0=9F=94=92=20substratebridge:=20refus?= =?UTF-8?q?e=20tar=20entries=20that=20write=20through=20symlinks=20on=20Un?= =?UTF-8?q?pack=20(#17813)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔒 substrate: unpack pulled workspaces through os.Root so tar symlinks cannot escape Fixes #17769 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * chore: re-trigger required checks after base moved Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-substrate-unpack-symlink.md | 1 + src/crux/sandbox/substratebridge/bridge.go | 35 ++++++++++++------- .../sandbox/substratebridge/bridge_test.go | 30 ++++++++++++++++ 3 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 changelog.d/security-substrate-unpack-symlink.md diff --git a/changelog.d/security-substrate-unpack-symlink.md b/changelog.d/security-substrate-unpack-symlink.md new file mode 100644 index 0000000000..28e56900e6 --- /dev/null +++ b/changelog.d/security-substrate-unpack-symlink.md @@ -0,0 +1 @@ +- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go index 653616887e..6804a7ba3b 100644 --- a/src/crux/sandbox/substratebridge/bridge.go +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -267,7 +267,10 @@ func Pack(dir string, w io.Writer) error { return gz.Close() } -// Unpack reads a Pack stream into dir, refusing entries that would escape it. +// Unpack reads a Pack stream into dir, refusing entries that escape it +// lexically (absolute or ".." names) or through symlinks: every write goes +// through an os.Root, so a symlink entry may be created but nothing is ever +// written through one that leaves dir. func Unpack(r io.Reader, dir string) error { gz, err := gzip.NewReader(r) if err != nil { @@ -277,6 +280,11 @@ func Unpack(r io.Reader, dir string) error { if err := os.MkdirAll(dir, 0o755); err != nil { return err } + root, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer root.Close() tr := tar.NewReader(gz) for { hdr, err := tr.Next() @@ -286,34 +294,34 @@ func Unpack(r io.Reader, dir string) error { if err != nil { return fmt.Errorf("tar: %w", err) } - target, err := safeJoin(dir, hdr.Name) + rel, err := safeRel(hdr.Name) if err != nil { return err } - if err := unpackEntry(tr, hdr, target); err != nil { - return err + if err := unpackEntry(tr, hdr, root, rel); err != nil { + return fmt.Errorf("tar: entry %q: %w", hdr.Name, err) } } } -func unpackEntry(tr *tar.Reader, hdr *tar.Header, target string) error { +func unpackEntry(tr *tar.Reader, hdr *tar.Header, root *os.Root, rel string) error { switch hdr.Typeflag { case tar.TypeDir: - return os.MkdirAll(target, os.FileMode(hdr.Mode)&os.ModePerm|0o700) + return root.MkdirAll(rel, os.FileMode(hdr.Mode)&os.ModePerm|0o700) case tar.TypeSymlink: - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { return err } - _ = os.Remove(target) - return os.Symlink(hdr.Linkname, target) + _ = root.Remove(rel) + return root.Symlink(hdr.Linkname, rel) case tar.TypeReg: if hdr.Size > maxTarEntry { return fmt.Errorf("tar: entry %q is too large (%d bytes)", hdr.Name, hdr.Size) } - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { return err } - f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) + f, err := root.OpenFile(rel, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) if err != nil { return err } @@ -328,12 +336,13 @@ func unpackEntry(tr *tar.Reader, hdr *tar.Header, target string) error { } } -func safeJoin(dir, name string) (string, error) { +// safeRel is the lexical first filter; os.Root enforces the rest. +func safeRel(name string) (string, error) { clean := filepath.Clean(filepath.FromSlash(name)) if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("tar: entry %q escapes the workspace", name) } - return filepath.Join(dir, clean), nil + return clean, nil } // Serve runs the bridge on ln until ctx ends. The actor's command is diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go index bc18a77a56..5483abc2a9 100644 --- a/src/crux/sandbox/substratebridge/bridge_test.go +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -154,6 +154,36 @@ func TestUnpackSymlinkReplacesExistingAndNestedEntries(t *testing.T) { } } +func TestUnpackRefusesWriteThroughSymlink(t *testing.T) { + for name, linkname := range map[string]func(outside string) string{ + "absolute": func(outside string) string { return outside }, + "relative": func(string) string { return "../outside-rel" }, + } { + t.Run(name, func(t *testing.T) { + parent := t.TempDir() + dst := filepath.Join(parent, "ws") + outside := t.TempDir() + if name == "relative" { + outside = filepath.Join(parent, "outside-rel") + if err := os.Mkdir(outside, 0o755); err != nil { + t.Fatal(err) + } + } + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: linkname(outside), Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "link/pwned", Typeflag: tar.TypeReg, Size: 2, Mode: 0o644}) + _, _ = tw.Write([]byte("no")) + }) + if err := Unpack(stream, dst); err == nil { + t.Fatal("write through escaping symlink succeeded") + } + if _, err := os.Stat(filepath.Join(outside, "pwned")); !os.IsNotExist(err) { + t.Errorf("file written outside the workspace: %v", err) + } + }) + } +} + func TestUnpackEntryErrorsWhenParentIsAFile(t *testing.T) { dst := t.TempDir() writeFile(t, filepath.Join(dst, "blocker"), "x") From 6fedca09074c6b18ed747de4441440e1e0a4f1d3 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:30:43 +0000 Subject: [PATCH 041/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20show=20substra?= =?UTF-8?q?te=20kicks=20and=20stream=20bridge=20output=20in=20the=20termin?= =?UTF-8?q?al=20pane=20(#17809)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: show substrate kicks and stream bridge output in the terminal pane Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 substratebridge: create the workspace dir before the streaming run test newBridgeServer intentionally leaves the workspace path uncreated so the PUT/GET workspace tests can exercise the missing-directory paths, but TestHandlerStreamsRunOutput ran a command in it and failed with chdir: no such file or directory before any frame was streamed. Hive-Fix-Attempt: 1/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 substrate kicks: stream only transitions after the viewer connected; cover new sandbox paths Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../added-terminal-pane-substrate-kicks.md | 1 + src/crux/sandbox/substrate.go | 41 ++- src/crux/sandbox/substrate_bridge.go | 50 ++- src/crux/sandbox/substrate_client_test.go | 10 +- src/crux/sandbox/substrate_kicks.go | 243 +++++++++++++ src/crux/sandbox/substrate_kicks_test.go | 331 ++++++++++++++++++ src/crux/sandbox/substrate_test.go | 17 +- src/crux/sandbox/substratebridge/bridge.go | 85 ++++- .../sandbox/substratebridge/bridge_test.go | 54 +++ src/docs/sandbox-isolation.md | 2 +- src/pkg/dashboard/job_terminal.go | 130 ++++++- .../dashboard/job_terminal_substrate_test.go | 250 +++++++++++++ .../dashboard/static-src/js/agent-cards.js | 11 +- src/pkg/dashboard/static/index.html | 11 +- 14 files changed, 1200 insertions(+), 36 deletions(-) create mode 100644 changelog.d/added-terminal-pane-substrate-kicks.md create mode 100644 src/crux/sandbox/substrate_kicks.go create mode 100644 src/crux/sandbox/substrate_kicks_test.go create mode 100644 src/pkg/dashboard/job_terminal_substrate_test.go diff --git a/changelog.d/added-terminal-pane-substrate-kicks.md b/changelog.d/added-terminal-pane-substrate-kicks.md new file mode 100644 index 0000000000..1514ab649f --- /dev/null +++ b/changelog.d/added-terminal-pane-substrate-kicks.md @@ -0,0 +1 @@ +- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go index 9ad2d81f2d..e79f4a87d7 100644 --- a/src/crux/sandbox/substrate.go +++ b/src/crux/sandbox/substrate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "strings" "time" @@ -174,7 +175,8 @@ type substrateControl interface { type substrateBridge interface { WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error - Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) + // Run copies the kick's output to live (nil allowed) as it arrives. + Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error } @@ -192,6 +194,16 @@ type SubstrateLauncher struct { newClient func(SubstrateOptions) (substrateClient, error) // now overrides the clock in tests. now func() time.Time + // kicks is where each kick's actor states and output are recorded for + // the dashboard's terminal pane; nil means DefaultSubstrateKicks. + kicks *SubstrateKicks +} + +func (l *SubstrateLauncher) kickRecord() *SubstrateKicks { + if l.kicks != nil { + return l.kicks + } + return DefaultSubstrateKicks } func (l *SubstrateLauncher) client() (substrateClient, error) { @@ -277,14 +289,18 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e } cleanupTemplate := func() { _ = control.DeleteActorTemplate(live(), ref) } - if _, err := control.CreateActor(ctx, &substrateapi.Actor{ + created, err := control.CreateActor(ctx, &substrateapi.Actor{ Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: name}, ActorTemplate: ref, WorkerSelector: substrateSelector(l.Options.WorkerSelector), - }); err != nil { + }) + if err != nil { cleanupTemplate() return res, fmt.Errorf("substrate: creating actor %s: %w", name, err) } + kicks := l.kickRecord() + kick := kicks.begin(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace, State: SubstrateStateCreated}) + kicks.observe(kick, created) // Delete is the default: a kick is a one-shot and keeping a snapshot // around costs object storage. snapshot_on_idle keeps a resumable // snapshot instead, so the next kick's warm state (caches, builds) is @@ -292,11 +308,15 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e cleanupActor := func() { if l.Options.SnapshotOnIdle && ctx.Err() == nil { if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { + kicks.update(kick, func(i *SubstrateActorInfo) { + i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true + }) cleanupTemplate() return } } _ = control.DeleteActor(live(), ref) + kicks.update(kick, func(i *SubstrateActorInfo) { i.State, i.Done = SubstrateStateDeleting, true }) cleanupTemplate() } @@ -314,7 +334,9 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e cleanupActor() return res, fmt.Errorf("substrate: resuming actor %s: %w", name, err) } - if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll); err != nil { + kicks.update(kick, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + observe := func(a *substrateapi.Actor) { kicks.observe(kick, a) } + if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll, observe); err != nil { cleanupActor() if ctx.Err() != nil { return res, ctx.Err() @@ -336,9 +358,10 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e if workDir == "" { workDir = SubstrateWorkspaceMount } - out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}) + out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}, kicks.output(kick)) if runErr == nil { res.Stdout, res.Stderr, res.ExitCode = out.Stdout, out.Stderr, out.ExitCode + kicks.update(kick, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, out.ExitCode }) if out.Error != "" { runErr = errors.New(out.Error) } @@ -364,8 +387,9 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e // substrateWaitRunning polls GetActor until the actor is RUNNING, or fails // when it crashes. A deleted or suspended actor during the wait is a failure -// too: nothing hive did asked for it. -func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration) error { +// too: nothing hive did asked for it. observe (nil allowed) sees every +// actor GetActor returns. +func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration, observe func(*substrateapi.Actor)) error { ticker := time.NewTicker(poll) defer ticker.Stop() for { @@ -373,6 +397,9 @@ func substrateWaitRunning(ctx context.Context, control substrateControl, ref *su if err != nil { return err } + if observe != nil { + observe(actor) + } switch state := actor.GetStatus().GetState(); state { case substrateapi.ActorState_ACTOR_STATE_RUNNING: return nil diff --git a/src/crux/sandbox/substrate_bridge.go b/src/crux/sandbox/substrate_bridge.go index 77f3ca4a16..5762b39bc8 100644 --- a/src/crux/sandbox/substrate_bridge.go +++ b/src/crux/sandbox/substrate_bridge.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -52,9 +53,14 @@ func (b *routerBridge) Push(ctx context.Context, ref *substrateapi.ObjectRef, wo return resp.Body.Close() } -// Run executes the kick command in the actor and waits for it to exit. -func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { +// Run executes the kick command in the actor and waits for it to exit. It +// asks the bridge to stream (RunRequest.Stream) and copies stdout and +// stderr to live as they arrive, so the dashboard's terminal pane can follow +// the kick; the returned RunResponse still carries the whole (capped) +// output for the transcript. live may be nil. +func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) { var out substratebridge.RunResponse + req.Stream = true body, err := json.Marshal(req) if err != nil { return out, err @@ -64,10 +70,44 @@ func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req return out, err } defer resp.Body.Close() - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return out, fmt.Errorf("decoding run response: %w", err) + return substrateDecodeRun(resp.Body, live) +} + +// substrateDecodeRun reads a /run answer: Partial frames from a streaming +// bridge, then the final frame with the exit code. A bridge that does not +// stream answers with one complete, non-Partial RunResponse, which is read +// the same way. +func substrateDecodeRun(r io.Reader, live io.Writer) (substratebridge.RunResponse, error) { + var out substratebridge.RunResponse + var stdout, stderr bytes.Buffer + dec := json.NewDecoder(r) + for { + var frame substratebridge.RunResponse + if err := dec.Decode(&frame); err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return out, fmt.Errorf("decoding run response: %w", err) + } + substrateAppendCapped(&stdout, frame.Stdout) + substrateAppendCapped(&stderr, frame.Stderr) + if live != nil { + _, _ = io.WriteString(live, frame.Stdout) + _, _ = io.WriteString(live, frame.Stderr) + } + if !frame.Partial { + out.Stdout, out.Stderr = stdout.String(), stderr.String() + out.ExitCode, out.Error = frame.ExitCode, frame.Error + return out, nil + } + } +} + +func substrateAppendCapped(buf *bytes.Buffer, s string) { + if room := substratebridge.MaxOutput - buf.Len(); len(s) > room { + s = s[:max(room, 0)] } - return out, nil + buf.WriteString(s) } // Pull replaces the local workspace with the actor's, so the commits the diff --git a/src/crux/sandbox/substrate_client_test.go b/src/crux/sandbox/substrate_client_test.go index fa9f0c8ea4..5a5638de27 100644 --- a/src/crux/sandbox/substrate_client_test.go +++ b/src/crux/sandbox/substrate_client_test.go @@ -394,10 +394,14 @@ func TestRouterBridgeDrivesTheBridgeProtocol(t *testing.T) { if got, _ := os.ReadFile(filepath.Join(actorWS, "sub", "file.txt")); string(got) != "before" { t.Fatalf("actor workspace = %q", got) } - out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}) + var live strings.Builder + out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}, &live) if err != nil || out.ExitCode != 4 || !strings.Contains(out.Stdout, "ran") { t.Fatalf("Run: %+v %v", out, err) } + if live.String() != "ran\n" { + t.Errorf("streamed output = %q", live.String()) + } if err := b.Pull(ctx, ref, local); err != nil { t.Fatalf("Pull: %v", err) } @@ -427,7 +431,7 @@ func TestRouterBridgeErrors(t *testing.T) { if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "unpack failed") { t.Errorf("non-2xx: %v", err) } - if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil { + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}, nil); err == nil { t.Error("Run swallowed a non-2xx") } if err := b.Pull(ctx, ref, t.TempDir()); err == nil { @@ -438,7 +442,7 @@ func TestRouterBridgeErrors(t *testing.T) { _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("{not json")) })) - if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil || !strings.Contains(err.Error(), "decoding run response") { + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}, nil); err == nil || !strings.Contains(err.Error(), "decoding run response") { t.Errorf("bad json: %v", err) } ws := filepath.Join(t.TempDir(), "ws") diff --git a/src/crux/sandbox/substrate_kicks.go b/src/crux/sandbox/substrate_kicks.go new file mode 100644 index 0000000000..1dffb978de --- /dev/null +++ b/src/crux/sandbox/substrate_kicks.go @@ -0,0 +1,243 @@ +package sandbox + +import ( + "context" + "io" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Substrate kick reader (hivecommons/hive#17777), the substrate counterpart +// of the OpenShell log reader in openshell_logs.go. +// +// A substrate-runtime agent has no tmux session, so the dashboard's +// read-only kick viewer needs the agent's live actor and its output. The +// Control API has no ListActors and Substrate has no logs RPC, so neither +// can come from the control plane: the launcher, which runs in the same +// hive process as the dashboard, records each kick here instead — the +// actor's name, every state it is seen in (RESUMING, RUNNING, then +// SUSPENDED or DELETING), the worker's SPIFFE id, and the kick's stdout +// and stderr as the bridge streams /run. Unlike the OpenShell gateway log, +// this is the kick's own output and the exit code is the kick's. + +// SubstrateActorInfo is what the viewer needs to know about a kick's actor. +type SubstrateActorInfo struct { + Name string + Atespace string + // State is the actor's ActorState without its ACTOR_STATE_ prefix, or + // CREATED before the control plane has reported one. + State string + // SpiffeID is the identity of the worker the actor was scheduled on + // (WorkerAssignment.worker_spiffe_id), once known. + SpiffeID string + // Exited is true once the bridge's /run reported the kick's exit code. + Exited bool + ExitCode int + // Suspended is true when the launcher kept a snapshot of the actor + // (snapshot_on_idle) instead of deleting it. + Suspended bool + // Done is true once the launcher has finished with the actor. + Done bool +} + +// Finished reports whether the kick will produce no more output. +func (i SubstrateActorInfo) Finished() bool { return i.Done } + +// SubstrateLogReader is the dashboard's view of the substrate kicks this +// process runs. *SubstrateKicks implements it. +type SubstrateLogReader interface { + // LatestActor returns the newest kick the launcher recorded for + // launchName (see LaunchName). A finished kick is still returned so the + // viewer can say how its actor ended; found is false when the agent has + // not run a substrate kick since hive started. + LatestActor(ctx context.Context, launchName string) (info SubstrateActorInfo, found bool, err error) + // StreamRun writes the kick's output to w, calling onState for every + // state the actor reaches after the call (the caller has already shown + // the one LatestActor reported), polling every poll + // until the launcher is done with the actor or ctx ends, and returns + // the last state it observed. + StreamRun(ctx context.Context, actorName string, poll time.Duration, w io.Writer, onState func(SubstrateActorInfo)) (SubstrateActorInfo, error) +} + +// DefaultSubstrateKicks is the process-wide record the launcher writes and +// the dashboard reads. +var DefaultSubstrateKicks = &SubstrateKicks{} + +var _ SubstrateLogReader = (*SubstrateKicks)(nil) + +// SubstrateKicks records the latest substrate kick per launch name. +type SubstrateKicks struct { + mu sync.Mutex + byLaunch map[string]*substrateKick +} + +// substrateKick is one recorded kick: every state snapshot it went through +// and its output so far, capped like the bridge's. +type substrateKick struct { + history []SubstrateActorInfo + out []byte +} + +func (k *substrateKick) latest() SubstrateActorInfo { return k.history[len(k.history)-1] } + +// begin records a new kick for launchName, replacing the previous one. +func (s *SubstrateKicks) begin(launchName string, info SubstrateActorInfo) *substrateKick { + s.mu.Lock() + defer s.mu.Unlock() + if s.byLaunch == nil { + s.byLaunch = map[string]*substrateKick{} + } + k := &substrateKick{history: []SubstrateActorInfo{info}} + s.byLaunch[launchName] = k + return k +} + +// update applies fn to the kick's latest state and records the result as +// a new snapshot when anything changed. +func (s *SubstrateKicks) update(k *substrateKick, fn func(*SubstrateActorInfo)) { + if k == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + next := k.latest() + fn(&next) + if next != k.latest() { + k.history = append(k.history, next) + } +} + +// observe records what GetActor reported: state and worker SPIFFE id. +func (s *SubstrateKicks) observe(k *substrateKick, actor *substrateapi.Actor) { + s.update(k, func(i *SubstrateActorInfo) { + if st := substrateStateName(actor.GetStatus().GetState()); st != "" { + i.State = st + } + if id := actor.GetStatus().GetWorkerAssignment().GetWorkerSpiffeId(); id != "" { + i.SpiffeID = id + } + }) +} + +// output returns a writer that appends to the kick's recorded output. +func (s *SubstrateKicks) output(k *substrateKick) io.Writer { + if k == nil { + return nil + } + return substrateKickWriter{s: s, k: k} +} + +type substrateKickWriter struct { + s *SubstrateKicks + k *substrateKick +} + +func (w substrateKickWriter) Write(p []byte) (int, error) { + w.s.mu.Lock() + defer w.s.mu.Unlock() + chunk := p + if room := substratebridge.MaxOutput - len(w.k.out); len(chunk) > room { + chunk = chunk[:max(room, 0)] + } + w.k.out = append(w.k.out, chunk...) + return len(p), nil +} + +// LatestActor implements SubstrateLogReader. +func (s *SubstrateKicks) LatestActor(_ context.Context, launchName string) (SubstrateActorInfo, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + k, ok := s.byLaunch[launchName] + if !ok { + return SubstrateActorInfo{}, false, nil + } + return k.latest(), true, nil +} + +func (s *SubstrateKicks) byActor(name string) *substrateKick { + s.mu.Lock() + defer s.mu.Unlock() + for _, k := range s.byLaunch { + if k.history[0].Name == name { + return k + } + } + return nil +} + +// StreamRun implements SubstrateLogReader. +func (s *SubstrateKicks) StreamRun(ctx context.Context, actorName string, poll time.Duration, w io.Writer, onState func(SubstrateActorInfo)) (SubstrateActorInfo, error) { + k := s.byActor(actorName) + if k == nil { + // Replaced by a newer kick before the viewer got here: this one is over. + return SubstrateActorInfo{Name: actorName, Done: true}, nil + } + ticker := time.NewTicker(poll) + defer ticker.Stop() + // The caller has just rendered LatestActor's snapshot, the newest one: + // replaying older history would show the actor going backwards. + s.mu.Lock() + seenStates, seenOut := len(k.history), 0 + s.mu.Unlock() + for { + s.mu.Lock() + states := append([]SubstrateActorInfo(nil), k.history[seenStates:]...) + out := append([]byte(nil), k.out[seenOut:]...) + seenStates, seenOut = len(k.history), len(k.out) + latest := k.latest() + s.mu.Unlock() + // States reached before the kick exited come before this poll's + // output, the rest after it, so the pane reads in order. + split := len(states) + for i, st := range states { + if st.Exited || st.Done { + split = i + break + } + } + emit := func(sts []SubstrateActorInfo) { + if onState != nil { + for _, st := range sts { + onState(st) + } + } + } + emit(states[:split]) + if len(out) > 0 { + if _, err := w.Write(out); err != nil { + return latest, err + } + } + emit(states[split:]) + if latest.Done { + return latest, nil + } + select { + case <-ctx.Done(): + return latest, ctx.Err() + case <-ticker.C: + } + } +} + +// substrateStateName is an ActorState as the viewer shows it: RUNNING, not +// ACTOR_STATE_RUNNING; empty for UNSPECIFIED. +func substrateStateName(st substrateapi.ActorState) string { + if st == substrateapi.ActorState_ACTOR_STATE_UNSPECIFIED { + return "" + } + return strings.TrimPrefix(st.String(), "ACTOR_STATE_") +} + +// Substrate actor states as SubstrateActorInfo.State carries them. +const ( + SubstrateStateCreated = "CREATED" + SubstrateStateResuming = "RESUMING" + SubstrateStateRunning = "RUNNING" + SubstrateStateSuspended = "SUSPENDED" + SubstrateStateDeleting = "DELETING" +) diff --git a/src/crux/sandbox/substrate_kicks_test.go b/src/crux/sandbox/substrate_kicks_test.go new file mode 100644 index 0000000000..50857ad705 --- /dev/null +++ b/src/crux/sandbox/substrate_kicks_test.go @@ -0,0 +1,331 @@ +package sandbox + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +func substrateStates(t *testing.T, kicks *SubstrateKicks, launchName string) ([]SubstrateActorInfo, string) { + t.Helper() + kicks.mu.Lock() + defer kicks.mu.Unlock() + k, ok := kicks.byLaunch[launchName] + if !ok { + t.Fatalf("no kick recorded for %q", launchName) + } + return append([]SubstrateActorInfo(nil), k.history...), string(k.out) +} + +func TestSubstrateLauncherRecordsKickForTerminal(t *testing.T) { + fake := &fakeSubstrate{ + spiffeID: "spiffe://ate.local/ns/ate-workers/sa/worker", + runResp: substratebridge.RunResponse{Stdout: "step 1\n", Stderr: "warn\n", ExitCode: 5}, + } + l := newTestSubstrate(fake) + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + history, out := substrateStates(t, l.kicks, "deps-sandbox") + if out != "step 1\nwarn\n" { + t.Errorf("recorded output = %q", out) + } + var states []string + for _, h := range history { + states = append(states, h.State) + } + if got := strings.Join(states, ","); got != "CREATED,RESUMING,RUNNING,RUNNING,DELETING" { + t.Errorf("states = %s", got) + } + final := history[len(history)-1] + if final.Name != "hive-kick-deps-1700000000" || final.Atespace != "hive" || final.SpiffeID != fake.spiffeID || + !final.Exited || final.ExitCode != 5 || !final.Done || final.Suspended { + t.Errorf("final = %+v", final) + } + if info, found, err := l.kicks.LatestActor(context.Background(), "deps-sandbox"); err != nil || !found || info != final { + t.Errorf("LatestActor = %+v %v %v", info, found, err) + } + if _, found, _ := l.kicks.LatestActor(context.Background(), "other-sandbox"); found { + t.Error("an agent without a kick must not be found") + } +} + +func TestSubstrateLauncherRecordsSuspendAndFailure(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + history, _ := substrateStates(t, l.kicks, "deps-sandbox") + if final := history[len(history)-1]; final.State != SubstrateStateSuspended || !final.Suspended || !final.Done { + t.Errorf("snapshot_on_idle final = %+v", final) + } + + fake = &fakeSubstrate{waitReadyErr: errors.New("no bridge")} + l = newTestSubstrate(fake) + if _, err := l.Run(context.Background(), substrateTestSpec()); err == nil { + t.Fatal("expected bridge error") + } + history, _ = substrateStates(t, l.kicks, "deps-sandbox") + if final := history[len(history)-1]; final.State != SubstrateStateDeleting || !final.Done || final.Exited { + t.Errorf("failed kick final = %+v", final) + } + + // A kick that fails before the actor exists records nothing. + fake = &fakeSubstrate{actorErr: errors.New("quota")} + l = newTestSubstrate(fake) + _, _ = l.Run(context.Background(), substrateTestSpec()) + if _, found, _ := l.kicks.LatestActor(context.Background(), "deps-sandbox"); found { + t.Error("no actor was created; nothing must be recorded") + } +} + +// substrateStreamWriter records what StreamRun writes and closes started on +// the first write, so a test knows the stream has begun polling before it +// records further transitions. +type substrateStreamWriter struct { + strings.Builder + started chan struct{} + once sync.Once +} + +func newSubstrateStreamWriter() *substrateStreamWriter { + return &substrateStreamWriter{started: make(chan struct{})} +} + +func (w *substrateStreamWriter) Write(p []byte) (int, error) { + defer w.once.Do(func() { close(w.started) }) + return w.Builder.Write(p) +} + +func substrateStateNames(infos []SubstrateActorInfo) string { + var states []string + for _, i := range infos { + states = append(states, i.State) + } + return strings.Join(states, ",") +} + +func TestSubstrateKicksStreamRunFollowsStatesAndOutput(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateCreated}) + kicks.update(k, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + w := kicks.output(k) + _, _ = w.Write([]byte("line 0\n")) + + type event struct{ state, out string } + var events []event + buf := newSubstrateStreamWriter() + done := make(chan SubstrateActorInfo) + go func() { + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, buf, func(i SubstrateActorInfo) { + events = append(events, event{state: i.State, out: buf.String()}) + }) + if err != nil { + t.Errorf("StreamRun: %v", err) + } + done <- final + }() + <-buf.started + kicks.observe(k, &substrateapi.Actor{Status: &substrateapi.ActorStatus{ + State: substrateapi.ActorState_ACTOR_STATE_RUNNING, + WorkerAssignment: &substrateapi.WorkerAssignment{WorkerSpiffeId: "spiffe://x/w"}, + }}) + _, _ = w.Write([]byte("line 1\n")) + _, _ = w.Write([]byte("line 2\n")) + kicks.update(k, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, 0 }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true }) + + final := <-done + if buf.String() != "line 0\nline 1\nline 2\n" { + t.Fatalf("streamed = %q", buf.String()) + } + if final.State != SubstrateStateSuspended || !final.Done || final.SpiffeID != "spiffe://x/w" { + t.Fatalf("final = %+v", final) + } + // RESUMING is what LatestActor reported before the stream started, so + // only the transitions recorded afterwards are reported. + var states []string + for _, e := range events { + states = append(states, e.state) + } + if got := strings.Join(states, ","); got != "RUNNING,RUNNING,SUSPENDED" { + t.Fatalf("states = %s", got) + } + if last := events[len(events)-1]; last.out != "line 0\nline 1\nline 2\n" { + t.Fatalf("SUSPENDED must be reported after the output, saw output %q", last.out) + } +} + +func TestSubstrateKicksStreamRunMidKickDoesNotReplayOlderStates(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateCreated}) + kicks.update(k, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.SpiffeID = SubstrateStateRunning, "spiffe://x/w" }) + w := kicks.output(k) + _, _ = w.Write([]byte("booting\n")) + + // The viewer connects while the kick is already RUNNING. + info, found, err := kicks.LatestActor(context.Background(), "deps-sandbox") + if err != nil || !found || info.State != SubstrateStateRunning { + t.Fatalf("LatestActor = %+v %v %v", info, found, err) + } + var seen []SubstrateActorInfo + buf := newSubstrateStreamWriter() + done := make(chan error) + go func() { + _, err := kicks.StreamRun(context.Background(), info.Name, time.Millisecond, buf, func(i SubstrateActorInfo) { + seen = append(seen, i) + }) + done <- err + }() + <-buf.started + kicks.update(k, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, 3 }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.Done = SubstrateStateDeleting, true }) + if err := <-done; err != nil { + t.Fatalf("StreamRun: %v", err) + } + for _, st := range seen { + if st.State == SubstrateStateCreated || st.State == SubstrateStateResuming { + t.Fatalf("a state older than RUNNING was replayed: %s", substrateStateNames(seen)) + } + } + if got := substrateStateNames(seen); got != "RUNNING,DELETING" { + t.Fatalf("states = %s", got) + } + if !seen[0].Exited || seen[0].ExitCode != 3 || !seen[1].Done { + t.Fatalf("transitions = %+v", seen) + } + if buf.String() != "booting\n" { + t.Fatalf("streamed = %q", buf.String()) + } +} + +func TestSubstrateKicksStreamRunWriteError(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + _, _ = kicks.output(k).Write([]byte("out")) + boom := errors.New("viewer gone") + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, failingWriter{err: boom}, nil) + if !errors.Is(err, boom) || final.State != SubstrateStateRunning || final.Done { + t.Fatalf("StreamRun = %+v %v", final, err) + } +} + +func TestSubstrateKicksStreamRunReplacedKick(t *testing.T) { + kicks := &SubstrateKicks{} + kicks.begin("other-sandbox", SubstrateActorInfo{Name: "b1", State: SubstrateStateRunning}) + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a2", State: SubstrateStateCreated}) + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, &strings.Builder{}, func(SubstrateActorInfo) { + t.Fatal("a replaced kick has no transitions to report") + }) + if err != nil || !final.Done || final.Name != "a1" { + t.Fatalf("replaced kick: %+v %v", final, err) + } +} + +func TestSubstrateLauncherDefaultsToProcessKicks(t *testing.T) { + if got := (&SubstrateLauncher{}).kickRecord(); got != DefaultSubstrateKicks { + t.Fatalf("kickRecord = %p, want DefaultSubstrateKicks", got) + } + own := &SubstrateKicks{} + if got := (&SubstrateLauncher{kicks: own}).kickRecord(); got != own { + t.Fatalf("kickRecord = %p, want %p", got, own) + } +} + +func TestSubstrateStateName(t *testing.T) { + if got := substrateStateName(substrateapi.ActorState_ACTOR_STATE_UNSPECIFIED); got != "" { + t.Errorf("UNSPECIFIED = %q", got) + } + if got := substrateStateName(substrateapi.ActorState_ACTOR_STATE_RUNNING); got != SubstrateStateRunning { + t.Errorf("RUNNING = %q", got) + } +} + +func TestSubstrateKicksStreamRunUnknownActorAndCancel(t *testing.T) { + kicks := &SubstrateKicks{} + final, err := kicks.StreamRun(context.Background(), "gone", time.Millisecond, &strings.Builder{}, nil) + if err != nil || !final.Done || final.Name != "gone" { + t.Fatalf("unknown actor: %+v %v", final, err) + } + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := kicks.StreamRun(ctx, "a1", time.Millisecond, &strings.Builder{}, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled stream: %v", err) + } +} + +func TestSubstrateKickOutputIsCapped(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("x", SubstrateActorInfo{Name: "a"}) + w := kicks.output(k) + big := make([]byte, substratebridge.MaxOutput+10) + if n, err := w.Write(big); err != nil || n != len(big) { + t.Fatalf("Write = %d %v", n, err) + } + if n, _ := w.Write([]byte("more")); n != 4 { + t.Fatal("a full buffer must still accept writes") + } + if len(k.out) != substratebridge.MaxOutput { + t.Fatalf("recorded %d bytes", len(k.out)) + } + if kicks.output(nil) != nil { + t.Fatal("no kick, no writer") + } + kicks.update(nil, func(*SubstrateActorInfo) { t.Fatal("must not run") }) +} + +func TestSubstrateDecodeRunStreamedAndLegacy(t *testing.T) { + var live strings.Builder + stream := `{"stdout":"a","stderr":"","exit_code":0,"partial":true} +{"stdout":"","stderr":"b","exit_code":0,"partial":true} +{"stdout":"","stderr":"","exit_code":2,"error":"boom"} +` + out, err := substrateDecodeRun(strings.NewReader(stream), &live) + if err != nil || out.Stdout != "a" || out.Stderr != "b" || out.ExitCode != 2 || out.Error != "boom" || live.String() != "ab" { + t.Fatalf("streamed: %+v %v live=%q", out, err, live.String()) + } + // A bridge that predates streaming answers with one complete response. + live.Reset() + out, err = substrateDecodeRun(strings.NewReader(`{"stdout":"all","stderr":"err","exit_code":1}`), &live) + if err != nil || out.Stdout != "all" || out.Stderr != "err" || out.ExitCode != 1 || live.String() != "allerr" { + t.Fatalf("legacy: %+v %v live=%q", out, err, live.String()) + } + // A stream cut before the final frame is an error, not exit code 0. + if _, err := substrateDecodeRun(strings.NewReader(`{"stdout":"a","partial":true}`), nil); err == nil || !strings.Contains(err.Error(), "decoding run response") { + t.Fatalf("truncated: %v", err) + } +} + +func TestSubstrateDecodeRunCapsOutput(t *testing.T) { + big := strings.Repeat("o", substratebridge.MaxOutput) + var stream strings.Builder + enc := json.NewEncoder(&stream) + for _, frame := range []substratebridge.RunResponse{ + {Stdout: big, Partial: true}, + {Stdout: "more", Stderr: "e", Partial: true}, + {ExitCode: 0}, + } { + if err := enc.Encode(frame); err != nil { + t.Fatal(err) + } + } + out, err := substrateDecodeRun(strings.NewReader(stream.String()), nil) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(out.Stdout) != substratebridge.MaxOutput || strings.Contains(out.Stdout, "more") || out.Stderr != "e" { + t.Fatalf("stdout %d bytes, stderr %q", len(out.Stdout), out.Stderr) + } +} diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go index f306ab9d39..1bdb458a75 100644 --- a/src/crux/sandbox/substrate_test.go +++ b/src/crux/sandbox/substrate_test.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "errors" + "io" "strings" "sync" "testing" @@ -32,6 +33,7 @@ type fakeSubstrate struct { getErr error states []substrateapi.ActorState crashMsg string + spiffeID string getCancels context.CancelFunc waitReadyErr error pushErr error @@ -98,7 +100,10 @@ func (f *fakeSubstrate) GetActor(ctx context.Context, ref *substrateapi.ObjectRe } return &substrateapi.Actor{ Metadata: &substrateapi.ResourceMetadata{Name: ref.GetName()}, - Status: &substrateapi.ActorStatus{State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}}, + Status: &substrateapi.ActorStatus{ + State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}, + WorkerAssignment: &substrateapi.WorkerAssignment{WorkerSpiffeId: f.spiffeID}, + }, }, nil } @@ -143,9 +148,12 @@ func (f *fakeSubstrate) Push(ctx context.Context, _ *substrateapi.ObjectRef, ws return f.pushErr } -func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { +func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) { f.note("Run", ctx) f.runReq = req + if live != nil { + _, _ = io.WriteString(live, f.runResp.Stdout+f.runResp.Stderr) + } if f.runCancels != nil { f.runCancels() } @@ -181,6 +189,7 @@ func newTestSubstrate(fake *fakeSubstrate) *SubstrateLauncher { }} l.newClient = func(SubstrateOptions) (substrateClient, error) { return fake, nil } l.now = func() time.Time { return time.Unix(1_700_000_000, 0) } + l.kicks = &SubstrateKicks{} return l } @@ -499,14 +508,14 @@ func TestSubstrateWaitRunningPollsUntilRunning(t *testing.T) { substrateapi.ActorState_ACTOR_STATE_RUNNING, }} ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} - if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err != nil { + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond, nil); err != nil { t.Fatalf("err = %v", err) } if n := len(fake.calls); n != 3 { t.Errorf("GetActor calls = %d, want 3", n) } fake = &fakeSubstrate{states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_SUSPENDING}} - if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond, nil); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { t.Errorf("err = %v", err) } } diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go index 6804a7ba3b..498b1aaa62 100644 --- a/src/crux/sandbox/substratebridge/bridge.go +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -24,6 +24,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" ) @@ -57,14 +58,24 @@ type RunRequest struct { Command []string `json:"command"` Env map[string]string `json:"env,omitempty"` WorkDir string `json:"workdir,omitempty"` + // Stream asks for the output as it is produced: the response is then a + // sequence of JSON RunResponse frames, every one but the last marked + // Partial and carrying only new output, the last carrying the exit + // code. A bridge that predates the field ignores it and answers with a + // single complete RunResponse, which a streaming client reads as a + // final frame, so either side can be upgraded first. + Stream bool `json:"stream,omitempty"` } -// RunResponse is the body POST /run answers with once the command exits. +// RunResponse is the body POST /run answers with once the command exits, +// or one frame of a streamed answer (see RunRequest.Stream). type RunResponse struct { Stdout string `json:"stdout"` Stderr string `json:"stderr"` ExitCode int `json:"exit_code"` Error string `json:"error,omitempty"` + // Partial marks a streamed output frame; its ExitCode means nothing. + Partial bool `json:"partial,omitempty"` } // Handler serves the bridge protocol for one workspace directory. @@ -159,16 +170,79 @@ func (h *Handler) run(w http.ResponseWriter, r *http.Request) { ctx, cancel = context.WithTimeout(ctx, h.RunTimeout) defer cancel() } - resp := Run(ctx, req, h.workspace()) w.Header().Set("Content-Type", contentTypeJSON) + if !req.Stream { + resp := Run(ctx, req, h.workspace()) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) + return + } w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(resp) + frames := &frameEncoder{enc: json.NewEncoder(w)} + if f, ok := w.(http.Flusher); ok { + frames.flush = f.Flush + } + resp := run(ctx, req, h.workspace(), &frameWriter{frames: frames, stderr: false}, &frameWriter{frames: frames, stderr: true}) + frames.send(RunResponse{ExitCode: resp.ExitCode, Error: resp.Error}) +} + +// frameEncoder serializes the streamed frames of one POST /run; stdout and +// stderr are written from separate goroutines by os/exec. +type frameEncoder struct { + mu sync.Mutex + enc *json.Encoder + flush func() +} + +func (f *frameEncoder) send(frame RunResponse) { + f.mu.Lock() + defer f.mu.Unlock() + _ = f.enc.Encode(frame) + if f.flush != nil { + f.flush() + } +} + +// frameWriter turns one of the command's output streams into Partial +// frames, capped at MaxOutput like the buffered answer. It never fails the +// command: a client that went away only loses the rest of the output. +type frameWriter struct { + frames *frameEncoder + stderr bool + written int +} + +func (w *frameWriter) Write(p []byte) (int, error) { + chunk := p + if room := MaxOutput - w.written; len(chunk) > room { + chunk = chunk[:max(room, 0)] + } + if len(chunk) > 0 { + w.written += len(chunk) + frame := RunResponse{Partial: true} + if w.stderr { + frame.Stderr = string(chunk) + } else { + frame.Stdout = string(chunk) + } + w.frames.send(frame) + } + return len(p), nil } // Run executes one RunRequest with the request env replacing the process // env — the launcher already filtered it through hive's credential-free // allowlist — and reports the exit code the way the Podman launcher does. func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { + var stdout, stderr bytes.Buffer + resp := run(ctx, req, workspace, &stdout, &stderr) + resp.Stdout, resp.Stderr = capString(stdout.Bytes()), capString(stderr.Bytes()) + return resp +} + +// run executes req with its output going to stdout and stderr and reports +// the exit code; the output fields of the result are left empty. +func run(ctx context.Context, req RunRequest, workspace string, stdout, stderr io.Writer) RunResponse { cmd := exec.CommandContext(ctx, req.Command[0], req.Command[1:]...) cmd.Dir = req.WorkDir if cmd.Dir == "" { @@ -187,10 +261,9 @@ func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { env = append(env, "PATH="+os.Getenv("PATH")) } cmd.Env = env - var stdout, stderr bytes.Buffer - cmd.Stdout, cmd.Stderr = &stdout, &stderr + cmd.Stdout, cmd.Stderr = stdout, stderr err := cmd.Run() - resp := RunResponse{Stdout: capString(stdout.Bytes()), Stderr: capString(stderr.Bytes())} + var resp RunResponse var exitErr *exec.ExitError switch { case err == nil: diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go index 5483abc2a9..c7e40bde59 100644 --- a/src/crux/sandbox/substratebridge/bridge_test.go +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -469,3 +469,57 @@ func TestPutWorkspaceClearsInPlace(t *testing.T) { t.Errorf("clearDir of a missing dir = %v, want nil", err) } } + +func TestHandlerStreamsRunOutput(t *testing.T) { + srv, ws := newBridgeServer(t) + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "echo one; echo two >&2; exit 2"}, Stream: true}) + resp, err := http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("run: %v %v", err, resp) + } + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + var stdout, stderr strings.Builder + var frames []RunResponse + for { + var f RunResponse + if err := dec.Decode(&f); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + frames = append(frames, f) + stdout.WriteString(f.Stdout) + stderr.WriteString(f.Stderr) + } + if len(frames) < 2 { + t.Fatalf("frames = %+v", frames) + } + for _, f := range frames[:len(frames)-1] { + if !f.Partial { + t.Fatalf("only the last frame may be final: %+v", frames) + } + } + if last := frames[len(frames)-1]; last.Partial || last.ExitCode != 2 || last.Stdout != "" || last.Stderr != "" { + t.Fatalf("final frame = %+v", last) + } + if stdout.String() != "one\n" || stderr.String() != "two\n" { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestFrameWriterCapsOutput(t *testing.T) { + var buf bytes.Buffer + w := &frameWriter{frames: &frameEncoder{enc: json.NewEncoder(&buf)}} + big := make([]byte, MaxOutput+5) + if n, err := w.Write(big); err != nil || n != len(big) { + t.Fatalf("Write = %d %v", n, err) + } + buf.Reset() + if n, err := w.Write([]byte("more")); err != nil || n != 4 || buf.Len() != 0 { + t.Fatalf("past the cap: n=%d err=%v frames=%q", n, err, buf.String()) + } +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 13199d6fee..4b9cf5a4fc 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -325,7 +325,7 @@ agents: **What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). -**The terminal pane.** A substrate-runtime agent has no tmux session and, until [#17699](https://github.com/hivecommons/hive/issues/17699), no live pane: the ▶ terminal shows the last persisted transcript like the job viewer between kicks. +**The terminal pane.** A substrate-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer behind the same handoff and role check ([#17777](https://github.com/hivecommons/hive/issues/17777)). The Control API has no `ListActors` and Substrate has no logs RPC, so the launcher — which runs in the same hive process as the dashboard — records each kick for the pane: the actor's `/`, every state it is seen in (`RESUMING`, `RUNNING`, then `SUSPENDED` with `snapshot_on_idle` or `DELETING`), and the worker's SPIFFE id from the actor's worker assignment. The bridge's `POST /run` streams the kick's stdout and stderr when asked (`"stream": true`, newline-delimited `RunResponse` frames; an older bridge answers with one complete response and still works), and the pane follows that output live, scrubbed of token-shaped values like the pod log, with each actor state change as a status line. Unlike OpenShell this is the kick's own output, and the pane ends with the kick's exit code and whether the actor was suspended or deleted. Between kicks it names how the last actor ended above the last persisted transcript; after a hive restart, before the agent's next kick, it shows the transcript alone. **Limitations (phase 2).** Workspace copied, not shared; `crux substrate-bridge` entrypoint and a published sandbox image land with the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); no resume of suspended actors; `api.github.com` not read-only at the policy layer; latency and image size not yet measured. diff --git a/src/pkg/dashboard/job_terminal.go b/src/pkg/dashboard/job_terminal.go index 032f93eff8..43800908c9 100644 --- a/src/pkg/dashboard/job_terminal.go +++ b/src/pkg/dashboard/job_terminal.go @@ -66,8 +66,20 @@ import ( // So while a kick runs the pane shows what the sandbox did around it // (network and filesystem decisions, supervisor state) and then points the // operator at the transcript, which is where the kick's own output and -// exit code land once the launcher's Exec().Run returns. Only the job pod -// log shows the kick's stdout live. +// exit code land once the launcher's Exec().Run returns. The job pod log +// and the substrate stream below show the kick's stdout live. +// +// Substrate runtime (hivecommons/hive#17777). An agent with +// sandbox.runtime: substrate runs each kick in a Substrate actor, again with +// no tmux session, so it uses the same page, route and three states. The +// Control API has no ListActors and Substrate no logs RPC, so the live +// source is sandbox.SubstrateLogReader: the launcher, in this same process, +// records the actor (name, atespace, every state it passes through, the +// worker's SPIFFE id) and tees the kick's stdout and stderr from the +// bridge's streamed /run into it. Unlike OpenShell this is the kick's own +// output and exit code. When the launcher is done the actor is deleted or, +// with snapshot_on_idle, suspended; the pane names which, and between kicks +// shows how the last actor ended above the persisted transcript. const ( // Both live under terminalPathPrefix so the terminal handoff code, @@ -99,6 +111,7 @@ type jobPodLogReader interface { type jobTerminalDeps struct { reader jobPodLogReader openshell sandbox.OpenShellLogReader + substrate sandbox.SubstrateLogReader workspaceRoot string pollInterval time.Duration } @@ -139,6 +152,15 @@ func (s *Server) openshellLogReader(name string) sandbox.OpenShellLogReader { return &sandbox.OpenShellLauncher{Options: opts} } +// substrateLogReader is the substrate counterpart: the process-wide record +// the substrate launcher writes each kick into. +func (s *Server) substrateLogReader() sandbox.SubstrateLogReader { + if s.jobTerminal.substrate != nil { + return s.jobTerminal.substrate + } + return sandbox.DefaultSubstrateKicks +} + func (s *Server) jobTerminalWorkspaceRoot() string { if s.jobTerminal.workspaceRoot != "" { return s.jobTerminal.workspaceRoot @@ -163,11 +185,11 @@ func (s *Server) jobTerminalPollInterval() time.Duration { // runtime runs its kicks outside the hive pod, so it has no tmux session // and its terminal is this read-only viewer. func sandboxRuntimeWithoutTmux(rt string) bool { - return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell + return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell || rt == config.SandboxRuntimeSubstrate } // jobTerminalAgent resolves the path's agent and checks that its effective -// sandbox runtime has no tmux session (job or openshell), returning that +// sandbox runtime has no tmux session (job, openshell or substrate), returning that // runtime. The status code says which precondition failed: 404 for an // unknown agent, 409 for one that has a tmux terminal instead. func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) { @@ -185,7 +207,7 @@ func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) } rt := agentCfg.SandboxRuntime(cfg.AgentSandbox) if !sandboxRuntimeWithoutTmux(rt) { - return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job or OpenShell sandbox; there is no sandbox log to show", name, rt) + return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job, OpenShell sandbox or Substrate actor; there is no sandbox log to show", name, rt) } return name, rt, http.StatusOK, "" } @@ -291,6 +313,10 @@ func (s *Server) handleJobTerminalStream(w http.ResponseWriter, r *http.Request) s.streamOpenShellSandbox(ctx, out, name, s.openshellLogReader(name)) return } + if rt == config.SandboxRuntimeSubstrate { + s.streamSubstrateActor(ctx, out, name, s.substrateLogReader()) + return + } reader := s.jobPodLogReader() pod, found, err := reader.LatestPod(ctx, sandbox.LaunchName(name)) @@ -407,6 +433,89 @@ func (s *Server) streamOpenShellSandbox(ctx context.Context, out *jobTerminalWri } } +// streamSubstrateActor is the substrate-runtime counterpart: name the +// agent's newest actor and its state and SPIFFE id, follow the kick's +// output as the bridge streams it, scrubbed line by line, with each actor +// state transition as a status line, then report the kick's exit code and +// whether the actor was suspended or deleted. A finished kick's actor is +// still named, above the persisted transcript. +func (s *Server) streamSubstrateActor(ctx context.Context, out *jobTerminalWriter, name string, reader sandbox.SubstrateLogReader) { + info, found, err := reader.LatestActor(ctx, sandbox.LaunchName(name)) + if err != nil { + out.status("cannot read Substrate actors for agent %q: %v", name, err) + out.status("showing the last persisted transcript instead") + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + if !found { + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + out.status("substrate actor %s", describeSubstrateActor(info)) + if info.Finished() { + out.status("%s", substrateActorEnd(info)) + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + out.status("%s", substrateLiveLogNote) + last := info + scrub := &kubejob.LineFilterWriter{Dst: out, Fn: scrubLogLine} + final, err := reader.StreamRun(ctx, info.Name, s.jobTerminalPollInterval(), scrub, func(next sandbox.SubstrateActorInfo) { + if next.State == last.State && next.SpiffeID == last.SpiffeID { + return + } + last = next + out.status("actor %s", describeSubstrateActor(next)) + }) + _ = scrub.Close() + if err != nil { + if ctx.Err() != nil { + return + } + out.status("output stream ended with an error: %v", err) + } + if final.Name == "" { + final.Name = info.Name + } + if final.Exited { + out.status("kick exited with code %d", final.ExitCode) + } + out.status("%s", substrateActorEnd(final)) +} + +// describeSubstrateActor is " · · ", leaving out +// what the launcher has not learned yet. +func describeSubstrateActor(i sandbox.SubstrateActorInfo) string { + parts := []string{i.Name, orUnknown(i.State)} + if i.Atespace != "" { + parts[0] = i.Atespace + "/" + i.Name + } + if i.SpiffeID != "" { + parts = append(parts, i.SpiffeID) + } + return strings.Join(parts, " · ") +} + +// substrateActorEnd says what became of the actor once the launcher was done +// with it, or that it has not finished. +func substrateActorEnd(i sandbox.SubstrateActorInfo) string { + switch { + case i.Suspended: + return fmt.Sprintf("actor %s was suspended with a snapshot (snapshot_on_idle): the kick has finished. %s", i.Name, substrateKickResultNote) + case i.Done: + return fmt.Sprintf("actor %s was deleted by the launcher: the kick has finished. %s", i.Name, substrateKickResultNote) + default: + return fmt.Sprintf("actor %s · %s (exit code not reported). %s", i.Name, orUnknown(i.State), substrateKickResultNote) + } +} + +const ( + // substrateLiveLogNote opens every live substrate stream. + substrateLiveLogNote = "this is the kick's stdout and stderr as the actor's bridge streams /run, scrubbed; actor state changes are shown as they happen" + // substrateKickResultNote closes it. + substrateKickResultNote = "the full output and exit code are in the persisted transcript; reopen this pane to see them" +) + const ( // openshellLiveLogNote opens every live OpenShell stream so an operator // never mistakes the sandbox's event log for the kick's output. @@ -440,11 +549,20 @@ var ( live: "OpenShell sandbox", where: "on the gateway", gone: "the launcher deletes a sandbox when its kick finishes", } + substrateTerms = sandboxTerms{ + title: "substrate actor log", + about: "the kick runs in a Substrate actor, so this is the kick's output as the actor's bridge streams it while it runs and the kick's transcript afterwards", + live: "Substrate actor", where: "started by this hive process", + gone: "the launcher deletes, or with snapshot_on_idle suspends, an actor when its kick finishes", + } ) func sandboxTermsFor(rt string) sandboxTerms { - if rt == config.SandboxRuntimeOpenShell { + switch rt { + case config.SandboxRuntimeOpenShell: return openshellTerms + case config.SandboxRuntimeSubstrate: + return substrateTerms } return jobTerms } diff --git a/src/pkg/dashboard/job_terminal_substrate_test.go b/src/pkg/dashboard/job_terminal_substrate_test.go new file mode 100644 index 0000000000..9db874f5f2 --- /dev/null +++ b/src/pkg/dashboard/job_terminal_substrate_test.go @@ -0,0 +1,250 @@ +package dashboard + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox" + "github.com/hivecommons/hive/pkg/config" +) + +// fakeSubstrateLogReader scripts sandbox.SubstrateLogReader: an optional +// recorded actor, the states it passes through, the output its bridge +// streams and the state it ends in. +type fakeSubstrateLogReader struct { + mu sync.Mutex + info sandbox.SubstrateActorInfo + found bool + latestErr error + states []sandbox.SubstrateActorInfo + lines []string + final sandbox.SubstrateActorInfo + streamErr error + launchNames []string + streamed []string + polls []time.Duration +} + +func (f *fakeSubstrateLogReader) LatestActor(_ context.Context, launchName string) (sandbox.SubstrateActorInfo, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.launchNames = append(f.launchNames, launchName) + return f.info, f.found, f.latestErr +} + +func (f *fakeSubstrateLogReader) StreamRun(_ context.Context, name string, poll time.Duration, w io.Writer, onState func(sandbox.SubstrateActorInfo)) (sandbox.SubstrateActorInfo, error) { + f.mu.Lock() + f.streamed = append(f.streamed, name) + f.polls = append(f.polls, poll) + states, lines, final, err := f.states, f.lines, f.final, f.streamErr + f.mu.Unlock() + for _, st := range states { + onState(st) + } + for _, l := range lines { + if _, werr := io.WriteString(w, l); werr != nil { + return final, werr + } + } + return final, err +} + +// substrateTerminalServer adds a substrate-runtime agent "actor" to the +// job-terminal fixture (which already has job "deps", podman "lint" and the +// tmux agent "scanner"). +func substrateTerminalServer(t *testing.T, reader *fakeSubstrateLogReader) (*Server, string) { + t.Helper() + s, root := jobTerminalServer(t, &fakeJobPodLogReader{}) + on := true + s.deps.Config.Agents["actor"] = config.AgentConfig{Backend: "claude", Enabled: true, Sandbox: &config.AgentSandboxOverride{Enabled: &on, Runtime: config.SandboxRuntimeSubstrate}} + s.jobTerminal.substrate = reader + return s, root +} + +const testSpiffeID = "spiffe://ate.local/ns/ate-workers/sa/worker" + +func TestJobTerminalGateAdmitsSubstrateRuntime(t *testing.T) { + if !sandboxRuntimeWithoutTmux(config.SandboxRuntimeSubstrate) { + t.Fatal("substrate agents have no tmux session; the viewer must admit them") + } + s, _ := substrateTerminalServer(t, &fakeSubstrateLogReader{}) + rec := doGet(s, "/terminal/job/actor") + if rec.Code != http.StatusOK { + t.Fatalf("substrate agent page: status = %d (%s)", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for _, want := range []string{"actor — substrate actor log", "Substrate actor", "'./actor/stream'"} { + if !strings.Contains(body, want) { + t.Errorf("substrate page missing %q", want) + } + } + if strings.Contains(body, "Kubernetes Job") || strings.Contains(body, "OpenShell") { + t.Fatal("substrate page must not describe the kick as a Job or OpenShell sandbox") + } + if rec := doGet(s, "/terminal/job/lint/stream"); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "Substrate actor") { + t.Fatalf("podman agent stream: status = %d (%s)", rec.Code, rec.Body.String()) + } +} + +func TestSubstrateTerminalStreamsKickWithActorTransitionsScrubbed(t *testing.T) { + token := "ghp_" + strings.Repeat("Z", 30) + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateResuming}, + found: true, + states: []sandbox.SubstrateActorInfo{ + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateRunning, SpiffeID: testSpiffeID}, + // Only the exit code changed: no new status line. + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateRunning, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4}, + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4, Suspended: true, Done: true}, + }, + lines: []string{"step 1\n", "auth " + token + " ok\n", "partial"}, + final: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4, Suspended: true, Done: true}, + } + s, _ := substrateTerminalServer(t, reader) + rec := doGet(s, "/terminal/job/actor/stream") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{ + "substrate actor hive/hive-kick-actor-1 · RESUMING", + "this is the kick's stdout and stderr as the actor's bridge streams /run", + "actor hive/hive-kick-actor-1 · RUNNING · " + testSpiffeID, + "actor hive/hive-kick-actor-1 · SUSPENDED · " + testSpiffeID, + "step 1\n", + "auth ghp_", + "partial", + // Unlike OpenShell, the bridge's exit code is the kick's own. + "kick exited with code 4", + "actor hive-kick-actor-1 was suspended with a snapshot (snapshot_on_idle): the kick has finished", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if n := strings.Count(body, "· RUNNING ·"); n != 1 { + t.Errorf("RUNNING reported %d times; a state is reported once per transition:\n%s", n, body) + } + if strings.Contains(body, token) { + t.Fatalf("live kick output reached the browser unscrubbed:\n%s", body) + } + reader.mu.Lock() + defer reader.mu.Unlock() + if len(reader.launchNames) != 1 || reader.launchNames[0] != sandbox.LaunchName("actor") { + t.Fatalf("actor must be looked up by the executor's launch name, got %v", reader.launchNames) + } + if len(reader.streamed) != 1 || reader.streamed[0] != "hive-kick-actor-1" || reader.polls[0] != time.Millisecond { + t.Fatalf("streamed %v at %v", reader.streamed, reader.polls) + } +} + +func TestSubstrateTerminalStreamReportsErrorAndDeletedActor(t *testing.T) { + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "a", State: sandbox.SubstrateStateRunning}, found: true, + final: sandbox.SubstrateActorInfo{Name: "a", State: sandbox.SubstrateStateDeleting, Done: true}, + streamErr: errors.New("router refused connect"), + } + s, _ := substrateTerminalServer(t, reader) + body := doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{"output stream ended with an error: router refused connect", "actor a was deleted by the launcher: the kick has finished. the full output and exit code are in the persisted transcript"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if strings.Contains(body, "kick exited with code") { + t.Fatalf("no exit code was reported; none must be shown:\n%s", body) + } + + reader = &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "b", State: sandbox.SubstrateStateRunning}, found: true, + final: sandbox.SubstrateActorInfo{Name: "b", State: sandbox.SubstrateStateRunning}, + } + s, _ = substrateTerminalServer(t, reader) + if body := doGet(s, "/terminal/job/actor/stream").Body.String(); !strings.Contains(body, "actor b · RUNNING (exit code not reported)") { + t.Fatalf("unfinished stream:\n%s", body) + } +} + +// Between kicks the recorded actor is finished: the pane names it and how +// it ended (suspended with snapshot_on_idle), then shows the transcript. +func TestSubstrateTerminalFinishedActorShowsTranscript(t *testing.T) { + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, Suspended: true, Done: true}, + found: true, + } + s, root := substrateTerminalServer(t, reader) + path := writeTranscript(t, root, "actor", 1, "stdout:\nfrom the last kick\n\nstderr:\n\n\n"+sandbox.TranscriptExitPrefix+"0\n") + body := doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{ + "substrate actor hive/hive-kick-actor-1 · SUSPENDED · " + testSpiffeID, + "actor hive-kick-actor-1 was suspended with a snapshot (snapshot_on_idle)", + path, "from the last kick", "last run exited with code 0", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + reader.mu.Lock() + streamed := reader.streamed + reader.mu.Unlock() + if len(streamed) != 0 { + t.Fatalf("finished actor: nothing must be streamed, got %v", streamed) + } +} + +func TestSubstrateTerminalFallsBackToTranscript(t *testing.T) { + reader := &fakeSubstrateLogReader{} + s, root := substrateTerminalServer(t, reader) + + body := doGet(s, "/terminal/job/actor/stream").Body.String() + if !strings.Contains(body, `no sandbox run yet for agent "actor": no Substrate actor started by this hive process`) || !strings.Contains(body, root) { + t.Fatalf("no-run state:\n%s", body) + } + + path := writeTranscript(t, root, "actor", 1, "stdout:\nfrom the last kick\n\nstderr:\n\n\n"+sandbox.TranscriptExitPrefix+"0\n") + body = doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{`no Substrate actor for agent "actor" right now`, path, "from the last kick", "last run exited with code 0"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + + reader.latestErr = errors.New("registry unavailable") + body = doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{"cannot read Substrate actors", "registry unavailable", "from the last kick"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } +} + +func TestSubstrateTerminalDefaultReaderIsLauncherRecord(t *testing.T) { + s, _ := apiServer(t) + if r, ok := s.substrateLogReader().(*sandbox.SubstrateKicks); !ok || r != sandbox.DefaultSubstrateKicks { + t.Fatal("default reader must be the record the substrate launcher writes") + } +} + +func TestStaticTerminalLinkRoutesSubstrateRuntimeToViewer(t *testing.T) { + body, err := os.ReadFile("static/index.html") + if err != nil { + t.Fatal(err) + } + html := string(body) + for _, want := range []string{ + "const SUBSTRATE_SANDBOX_RUNTIME = 'substrate';", + "|| sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME;", + "if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) {", + } { + if !strings.Contains(html, want) { + t.Fatalf("static dashboard substrate-terminal wiring missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index db730d57dd..5253df7de5 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -73,11 +73,15 @@ by the Go server under the same /terminal prefix so openTerminal()'s handoff code and assertion cookie apply unchanged. openshell agents (#17645) run each kick in a gateway-owned sandbox and use the same - viewer, which streams the sandbox's log instead. */ + viewer, which streams the sandbox's log instead; substrate agents + (#17777) run each kick in a Substrate actor and the viewer streams the + kick's output from the actor's bridge. */ const JOB_SANDBOX_RUNTIME = 'job'; const OPENSHELL_SANDBOX_RUNTIME = 'openshell'; + const SUBSTRATE_SANDBOX_RUNTIME = 'substrate'; function sandboxRuntimeWithoutTmux(sandboxRuntime) { - return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME; + return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME + || sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME; } function agentUsesJobTerminal(a) { return !!a && sandboxRuntimeWithoutTmux(a.sandboxRuntime); @@ -93,6 +97,9 @@ } function terminalLinkTitle(a, tmuxTitle) { if (!agentUsesJobTerminal(a)) return tmuxTitle; + if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) { + return 'Stream this agent\'s kick output and Substrate actor state (read-only: it runs in a Substrate actor, not a tmux session)'; + } return a.sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME ? 'Stream the log of this agent\'s OpenShell sandbox (read-only: it runs in a gateway-owned sandbox, not a tmux session)' : 'Stream the log of this agent\'s sandbox Job pod (read-only: it runs as a Kubernetes Job, not a tmux session)'; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index fd73ff80e2..21292a873d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -30271,11 +30271,15 @@

All labels

by the Go server under the same /terminal prefix so openTerminal()'s handoff code and assertion cookie apply unchanged. openshell agents (#17645) run each kick in a gateway-owned sandbox and use the same - viewer, which streams the sandbox's log instead. */ + viewer, which streams the sandbox's log instead; substrate agents + (#17777) run each kick in a Substrate actor and the viewer streams the + kick's output from the actor's bridge. */ const JOB_SANDBOX_RUNTIME = 'job'; const OPENSHELL_SANDBOX_RUNTIME = 'openshell'; + const SUBSTRATE_SANDBOX_RUNTIME = 'substrate'; function sandboxRuntimeWithoutTmux(sandboxRuntime) { - return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME; + return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME + || sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME; } function agentUsesJobTerminal(a) { return !!a && sandboxRuntimeWithoutTmux(a.sandboxRuntime); @@ -30291,6 +30295,9 @@

All labels

} function terminalLinkTitle(a, tmuxTitle) { if (!agentUsesJobTerminal(a)) return tmuxTitle; + if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) { + return 'Stream this agent\'s kick output and Substrate actor state (read-only: it runs in a Substrate actor, not a tmux session)'; + } return a.sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME ? 'Stream the log of this agent\'s OpenShell sandbox (read-only: it runs in a gateway-owned sandbox, not a tmux session)' : 'Stream the log of this agent\'s sandbox Job pod (read-only: it runs as a Kubernetes Job, not a tmux session)'; From 9761595af0c9e537e21cea89591835cd34e33716 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 18:33:35 -0700 Subject: [PATCH 042/223] =?UTF-8?q?=F0=9F=90=9B=20Mark=20hive's=20in-clust?= =?UTF-8?q?er=20k8s=20API=20dials=20so=20the=20egress=20redirect=20never?= =?UTF-8?q?=20MITMs=20them=20(#17818)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Mark hive's in-cluster k8s API dials so the egress redirect never MITMs them On nodes without the xt_owner netfilter module (IBM Cloud IKS/ROKS) the HIVE_PROXY chain has only the packet-mark self-exemption, so every :443 dial the hive makes for itself must carry SO_MARK. The proxy's upstream dials did; the spoke's own Kubernetes API clients (self-upgrade, startup probe, node metrics, ingress probes, kubejob) did not, so they were REDIRECTed into the MITM proxy, hit the private-range forward guard and got EOF — every wec2 hive failed self-upgrade with 'reading startup probe before upgrade … EOF'. Move the SO_MARK dialer into a new crux/egressmark package (crux/proxy imports crux/agent which imports kubejob, so kubejob cannot import proxy) and use it for all five in-cluster API transports. Signed-off-by: Andrew Anderson * 🌱 Allow crux/egressmark in the dashboard import ratchet Signed-off-by: Andrew Anderson * 🌱 Add crux/egressmark tests for the coverage gate Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../fixed-k8s-api-client-egress-mark.md | 1 + src/crux/egressmark/egressmark.go | 82 +++++++++++++++ src/crux/egressmark/egressmark_test.go | 84 ++++++++++++++++ src/crux/egressmark/somark_linux.go | 11 +++ src/crux/egressmark/somark_other.go | 9 ++ src/crux/kubejob/kubejob.go | 3 +- src/crux/proxy/github_proxy.go | 99 +++---------------- src/crux/proxy/somark_linux.go | 20 ---- src/crux/proxy/somark_other.go | 11 --- src/docs/net-admin-requirement.md | 2 + .../testutil/dashboard_import_allowlist.txt | 2 + src/pkg/dashboard/deployment_upgrade.go | 3 +- src/pkg/hub/spoke/cluster_metrics.go | 4 +- src/pkg/hub/spoke/heartbeat.go | 2 + src/pkg/hub/spoke/self_upgrade.go | 3 +- 15 files changed, 217 insertions(+), 119 deletions(-) create mode 100644 changelog.d/fixed-k8s-api-client-egress-mark.md create mode 100644 src/crux/egressmark/egressmark.go create mode 100644 src/crux/egressmark/egressmark_test.go create mode 100644 src/crux/egressmark/somark_linux.go create mode 100644 src/crux/egressmark/somark_other.go delete mode 100644 src/crux/proxy/somark_linux.go delete mode 100644 src/crux/proxy/somark_other.go diff --git a/changelog.d/fixed-k8s-api-client-egress-mark.md b/changelog.d/fixed-k8s-api-client-egress-mark.md new file mode 100644 index 0000000000..b33246cd5f --- /dev/null +++ b/changelog.d/fixed-k8s-api-client-egress-mark.md @@ -0,0 +1 @@ +- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). diff --git a/src/crux/egressmark/egressmark.go b/src/crux/egressmark/egressmark.go new file mode 100644 index 0000000000..0d70e8fb1d --- /dev/null +++ b/src/crux/egressmark/egressmark.go @@ -0,0 +1,82 @@ +// Package egressmark stamps the hive's OWN outbound sockets with the fwmark +// (SO_MARK) that the entrypoint's forced-egress iptables rule exempts +// (`-m mark --mark -j RETURN`), so control-plane traffic the hive +// itself originates is never redirected into its own MITM proxy. +// +// It is shared by the proxy's upstream dials and by every in-cluster +// Kubernetes API client (self-upgrade, startup-probe backfill, node metrics, +// ingress probes, kubejob). The entrypoint also exempts the hive's uid with +// `-m owner`, but that match needs the xt_owner kernel module, which IBM Cloud +// IKS/ROKS and other OVN nodes do not load: there the mark is the ONLY +// exemption, and an unmarked dial to kubernetes.default.svc:443 lands in the +// proxy, whose private-range forward guard closes it ("EOF" / "unexpected eof +// while reading"), failing every spoke self-upgrade on that cluster. +package egressmark + +import ( + "log/slog" + "net" + "os" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + // DefaultMark is the fwmark the entrypoint exempts when + // HIVE_PROXY_EGRESS_MARK is unset. entrypoint.sh is the single source of + // truth and exports the same value into the hive process. + DefaultMark = 0x1112 + + // Env is the variable entrypoint.sh exports so the Go side and the + // iptables rule stay in lockstep. + Env = "HIVE_PROXY_EGRESS_MARK" +) + +// Mark returns the fwmark to stamp on the hive's own dials, read from +// HIVE_PROXY_EGRESS_MARK (hex with 0x prefix or decimal). It falls back to +// DefaultMark when unset or unparseable: that default matches the entrypoint +// default, so accidental drift stays on the agreed value rather than silently +// disabling the exemption. +func Mark() int { + raw := strings.TrimSpace(os.Getenv(Env)) + if raw == "" { + return DefaultMark + } + if v, err := strconv.ParseInt(raw, 0, 64); err == nil && v > 0 { + return int(v) + } + return DefaultMark +} + +// Dialer returns a *net.Dialer whose Control hook stamps Mark() on the socket +// before connect. The optional timeout is applied when non-zero. +// +// The mark is BEST-EFFORT: setsockopt(SO_MARK) needs CAP_NET_ADMIN in the +// effective set, and a hive running without it must still dial (on OKE the +// owner-uid exemption covers it). A mark failure is therefore logged once and +// the dial proceeds unmarked instead of aborting. +func Dialer(timeout time.Duration) *net.Dialer { + mark := Mark() + return &net.Dialer{ + Timeout: timeout, + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + if err := setSockMark(fd, mark); err != nil { + warnOnce(mark, err) + } + }) + }, + } +} + +var warnSync sync.Once + +func warnOnce(mark int, err error) { + warnSync.Do(func() { + slog.Warn("egress SO_MARK unavailable — dialing unmarked (owner-UID exemption still applies where xt_owner is loaded); expected when the hive runs without CAP_NET_ADMIN in its effective set", + "mark", mark, "error", err) + }) +} diff --git a/src/crux/egressmark/egressmark_test.go b/src/crux/egressmark/egressmark_test.go new file mode 100644 index 0000000000..9c3ed7040d --- /dev/null +++ b/src/crux/egressmark/egressmark_test.go @@ -0,0 +1,84 @@ +package egressmark + +import ( + "errors" + "net" + "syscall" + "testing" + "time" +) + +// TestMarkDefaultAndEnvOverride verifies the mark resolves to the entrypoint +// default when unset, honours hex and decimal overrides, and fails safe back +// to the default on garbage or zero rather than disabling the exemption. +func TestMarkDefaultAndEnvOverride(t *testing.T) { + cases := []struct { + name string + env string + want int + }{ + {"unset", "", DefaultMark}, + {"hex", "0x2223", 0x2223}, + {"decimal", "8739", 8739}, + {"garbage falls back", "not-a-number", DefaultMark}, + {"zero falls back", "0", DefaultMark}, + {"negative falls back", "-5", DefaultMark}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(Env, tc.env) + if got := Mark(); got != tc.want { + t.Fatalf("Mark() with %s=%q = %#x, want %#x", Env, tc.env, got, tc.want) + } + }) + } +} + +// TestDialerControlDoesNotBreakDial dials a loopback listener through the +// marked dialer. The Control hook must run and attempt SO_MARK without failing +// the dial: the hive runs without CAP_NET_ADMIN in its effective set in many +// deployments, so setsockopt(SO_MARK) returns EPERM there and a propagated +// error would break every self-upgrade / startup-probe / kubejob API call. +func TestDialerControlDoesNotBreakDial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + }() + + d := Dialer(2 * time.Second) + if d.Timeout != 2*time.Second { + t.Fatalf("Dialer timeout = %v, want 2s", d.Timeout) + } + conn, err := d.Dial("tcp", ln.Addr().String()) + if err != nil { + if errors.Is(err, syscall.EPERM) { + t.Fatalf("Dialer aborted the dial on SO_MARK EPERM — mark must be best-effort: %v", err) + } + t.Fatalf("Dialer Dial failed (Control hook must not break the dial): %v", err) + } + conn.Close() +} + +// TestDialerZeroTimeoutLeavesDefault confirms a zero timeout is passed through +// unchanged so callers that rely on the http.Transport's own deadlines are not +// given a surprise dial timeout. +func TestDialerZeroTimeoutLeavesDefault(t *testing.T) { + if d := Dialer(0); d.Timeout != 0 || d.Control == nil { + t.Fatalf("Dialer(0) = timeout %v, control nil=%v; want 0 and a Control hook", d.Timeout, d.Control == nil) + } +} + +// TestWarnOnceIsIdempotent exercises the once-only warning path directly so a +// mark failure on every dial does not spam the log. +func TestWarnOnceIsIdempotent(t *testing.T) { + warnOnce(DefaultMark, errors.New("first")) + warnOnce(DefaultMark, errors.New("second")) +} diff --git a/src/crux/egressmark/somark_linux.go b/src/crux/egressmark/somark_linux.go new file mode 100644 index 0000000000..0e7ca6e503 --- /dev/null +++ b/src/crux/egressmark/somark_linux.go @@ -0,0 +1,11 @@ +//go:build linux + +package egressmark + +import "syscall" + +// setSockMark sets SO_MARK on the raw socket so the kernel stamps every packet +// it emits with the fwmark the entrypoint's nat/OUTPUT RETURN rule matches. +func setSockMark(fd uintptr, mark int) error { + return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, mark) +} diff --git a/src/crux/egressmark/somark_other.go b/src/crux/egressmark/somark_other.go new file mode 100644 index 0000000000..9cb20460a3 --- /dev/null +++ b/src/crux/egressmark/somark_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package egressmark + +// setSockMark is a no-op off Linux: SO_MARK is Linux-only, but the package must +// still compile on the CI matrix and developer machines. +func setSockMark(fd uintptr, mark int) error { + return nil +} diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index 5a7001bc59..bc24f16b9b 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -41,6 +41,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/sandbox" ) @@ -779,6 +780,6 @@ func (l *Launcher) client() (*http.Client, error) { if !pool.AppendCertsFromPEM(pem) { return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates") } - l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}} return l.HTTPClient, nil } diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 35a1ad0844..c47abfedf8 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -11,15 +11,14 @@ import ( "net" "net/http" "os" - "strconv" "strings" "sync" - "syscall" "time" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/inferencehealth" "github.com/hivecommons/hive/crux/upstreamproxy" ) @@ -38,94 +37,26 @@ const ( maxViolationLog = 1000 maxGitHubWriteBodyScan = 2 << 20 - // defaultProxyEgressMark is the fwmark (SO_MARK) the proxy stamps onto its - // OWN upstream :443 dials so the forced-egress iptables redirect can exempt - // them (via `-m mark --mark -j RETURN`) and avoid looping the proxy's - // traffic back into itself. - // - // This replaces the `-m owner --uid-owner` self-exemption, which needs the - // xt_owner kernel module — absent on OpenShift/OVN nodes. A packet mark works - // on both OKE and OpenShift. The value is arbitrary but MUST match the mark - // the entrypoint exempts; entrypoint.sh is the single source of truth and - // exports HIVE_PROXY_EGRESS_MARK (defaulting to this same value) into the - // proxy process, which reads it via proxyEgressMark(). - defaultProxyEgressMark = 0x1112 - - // proxyEgressMarkEnv is the env var the entrypoint exports to keep the proxy - // and the iptables rule in sync from one source of truth. - proxyEgressMarkEnv = "HIVE_PROXY_EGRESS_MARK" + // defaultProxyEgressMark and proxyEgressMarkEnv mirror crux/egressmark, + // the shared home of the SO_MARK self-exemption (also used by the hive's + // in-cluster Kubernetes API clients). entrypoint.sh remains the single + // source of truth for the value. + defaultProxyEgressMark = egressmark.DefaultMark + proxyEgressMarkEnv = egressmark.Env ) -// proxyEgressMark returns the fwmark to stamp on the proxy's upstream dials, -// read from HIVE_PROXY_EGRESS_MARK (set by entrypoint.sh) so the Go proxy and -// the iptables exemption stay in lockstep. Accepts hex (0x1112) or decimal. -// Falls back to defaultProxyEgressMark when unset or unparseable — that default -// matches the entrypoint default, so an accidental drift fails safe by staying -// on the agreed value rather than silently disabling the exemption. +// proxyEgressMark returns the fwmark to stamp on the proxy's upstream dials; +// see egressmark.Mark. func proxyEgressMark() int { - raw := strings.TrimSpace(os.Getenv(proxyEgressMarkEnv)) - if raw == "" { - return defaultProxyEgressMark - } - // ParseInt with base 0 honours a 0x prefix (hex) and plain decimal. - if v, err := strconv.ParseInt(raw, 0, 64); err == nil && v > 0 { - return int(v) - } - return defaultProxyEgressMark + return egressmark.Mark() } -// markDialer returns a *net.Dialer whose Control hook stamps the proxy egress -// fwmark (SO_MARK) on the outbound socket before connect. On non-Linux builds -// setSockMark is a no-op, so the dialer is portable. The optional timeout is -// applied when non-zero. -// -// SO_MARK is a BEST-EFFORT egress hint, never a hard requirement: it exists so -// the forced-egress iptables redirect can exempt the proxy's own upstream dials -// via `-m mark`. But setsockopt(SO_MARK) needs CAP_NET_ADMIN in the process's -// EFFECTIVE set, and the hive Go process runs as a non-root user (entrypoint -// drops from root to `dev` via gosu, which strips all capabilities). So the mark -// call fails with EPERM ("operation not permitted") on exactly the deployments -// that run non-root — which is all of them. Returning that error from the -// Control hook ABORTS the dial, so every proxy upstream dial fails and agents -// can't reach GitHub at all. That is a regression: on OKE the `-m owner -// --uid-owner` exemption already covers the proxy's uid dials, so an unmarked -// dial succeeds; on OpenShift (no xt_owner) the mark is the only exemption, but -// there too, failing the dial helps nothing. So we log the mark failure ONCE and -// proceed unmarked rather than killing the connection. +// markDialer returns a *net.Dialer whose sockets carry the proxy egress fwmark +// so the forced-egress REDIRECT exempts them. The mark is best-effort: without +// CAP_NET_ADMIN the dial proceeds unmarked (owner-UID exemption on OKE); see +// egressmark.Dialer for the full rationale. func markDialer(timeout time.Duration) *net.Dialer { - mark := proxyEgressMark() - return &net.Dialer{ - Timeout: timeout, - Control: func(_, _ string, c syscall.RawConn) error { - // A failure to enter the raw-conn control context is a real dial - // problem (the fd is unusable) — propagate it. - if err := c.Control(func(fd uintptr) { - if markErr := setSockMark(fd, mark); markErr != nil { - warnSockMarkOnce(mark, markErr) - } - }); err != nil { - return err - } - // Deliberately do NOT propagate a setSockMark failure: the mark is a - // best-effort exemption hint, and the owner-UID iptables RETURN - // already exempts the proxy's own uid on OKE, so an unmarked dial - // still reaches GitHub. - return nil - }, - } -} - -// warnSockMarkOnce logs the first SO_MARK failure at WARN and every subsequent -// one at DEBUG, so a non-CAP_NET_ADMIN deployment (the common non-root case) -// does not flood the log with one line per dial while still surfacing the -// condition once. -var sockMarkWarnOnce sync.Once - -func warnSockMarkOnce(mark int, err error) { - sockMarkWarnOnce.Do(func() { - slog.Warn("proxy egress SO_MARK unavailable — dialing unmarked (owner-UID exemption still applies on OKE); this is expected when the hive runs non-root without CAP_NET_ADMIN in its effective set", - "mark", mark, "error", err) - }) + return egressmark.Dialer(timeout) } // CACertPath and caKeyPath are the PVC locations of the persisted MITM CA. diff --git a/src/crux/proxy/somark_linux.go b/src/crux/proxy/somark_linux.go deleted file mode 100644 index b7b0158b68..0000000000 --- a/src/crux/proxy/somark_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build linux - -package proxy - -import ( - "syscall" -) - -// setSockMark sets SO_MARK on the raw socket file descriptor. The kernel then -// stamps every packet the socket emits with this fwmark, which the entrypoint's -// iptables nat/OUTPUT rule matches (-m mark --mark -j RETURN) to exempt -// the proxy's OWN upstream traffic from the :443 -> :18443 redirect, so it does -// not loop back into itself. -// -// Unlike `-m owner --uid-owner`, packet marks do NOT require the xt_owner kernel -// module, which is absent on OpenShift/OVN nodes. The mark exemption is the -// single self-exemption mechanism that works on both OKE and OpenShift. -func setSockMark(fd uintptr, mark int) error { - return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, mark) -} diff --git a/src/crux/proxy/somark_other.go b/src/crux/proxy/somark_other.go deleted file mode 100644 index e5fee60dc7..0000000000 --- a/src/crux/proxy/somark_other.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !linux - -package proxy - -// setSockMark is a no-op on non-Linux platforms: SO_MARK is a Linux-only socket -// option. The proxy only runs inside the Linux container, but the package must -// still compile on the CI build matrix (and on developer darwin machines), so -// this stub keeps the mark-setting dialer portable. -func setSockMark(fd uintptr, mark int) error { - return nil -} diff --git a/src/docs/net-admin-requirement.md b/src/docs/net-admin-requirement.md index d0a9e06078..aea764ad62 100644 --- a/src/docs/net-admin-requirement.md +++ b/src/docs/net-admin-requirement.md @@ -127,6 +127,8 @@ match no traffic even mid-probe) and is unconditionally torn down: | `xt_REDIRECT` | the `REDIRECT` target that forces `:443` through the proxy | **FATAL, exit 77** | | `xt_owner` | owner-UID self-exemption (`-m owner`) | WARN only — optional by design; OpenShift/OVN runs without it and the mark exemption carries the proxy's egress | +Without `xt_owner` the mark is the only exemption, so every outbound `:443` dial the hive makes for itself must carry it. The proxy's upstream dials and the in-cluster Kubernetes API clients (self-upgrade, startup-probe backfill, node metrics, ingress probes, kubejob) share `crux/egressmark.Dialer`; an unmarked dial to `kubernetes.default.svc` would be redirected into the proxy and closed by its private-range forward guard (`EOF`), which is how IBM Cloud IKS spokes failed every self-upgrade with `reading startup probe before upgrade … EOF`. + When a required module is missing, the FATAL names it explicitly: ``` diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 43b8a642cf..63c7035f19 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -8,9 +8,11 @@ # seam or be added here with PR justification; removed imports must be # deleted so the ratchet tightens. Entries are sorted. # workflowmodel: /api/workflows handlers compile lifecycle models (Compile, Profile diagnostics) and the tracker emits per-repo workflow columns; the store and bindings already route through Dependencies (hivecommons/hive#17620 P2). +# egressmark: deployment_upgrade.go dials the in-cluster Kubernetes API for the hive's own rollout; the dial must carry the forced-egress SO_MARK or nodes without xt_owner redirect it into the MITM proxy (hivecommons/hive#17817). github.com/hivecommons/hive/crux github.com/hivecommons/hive/crux/agentparse github.com/hivecommons/hive/crux/claude +github.com/hivecommons/hive/crux/egressmark github.com/hivecommons/hive/crux/kubejob github.com/hivecommons/hive/crux/logscrub github.com/hivecommons/hive/crux/paneclass diff --git a/src/pkg/dashboard/deployment_upgrade.go b/src/pkg/dashboard/deployment_upgrade.go index 1c15e90f3b..61f21bbec3 100644 --- a/src/pkg/dashboard/deployment_upgrade.go +++ b/src/pkg/dashboard/deployment_upgrade.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" ) @@ -356,7 +357,7 @@ func kubernetesSelfUpgradeRBACPrecheck() error { } client := &http.Client{ Timeout: selfUpgradePrecheckTimeout, - Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}}, + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}, } path := fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/%s", namespace, selfUpgradeDeploymentName) if err := kubernetesSelfUpgradeRequest(client, http.MethodGet, path, nil, token); err != nil { diff --git a/src/pkg/hub/spoke/cluster_metrics.go b/src/pkg/hub/spoke/cluster_metrics.go index aef70da225..031bc03a0c 100644 --- a/src/pkg/hub/spoke/cluster_metrics.go +++ b/src/pkg/hub/spoke/cluster_metrics.go @@ -14,6 +14,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/crux/egressmark" ) const ( @@ -547,7 +549,7 @@ func k8sAPIGetWithContext(ctx context.Context, path string) ([]byte, error) { client := &http.Client{ Timeout: metricsCollectionTimeout, - Transport: &http.Transport{TLSClientConfig: tlsConfig}, + Transport: &http.Transport{TLSClientConfig: tlsConfig, DialContext: egressmark.Dialer(0).DialContext}, } ctx, cancel := context.WithTimeout(ctx, metricsCollectionTimeout) diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 705efdbaa7..5ab3c0990e 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -23,6 +23,7 @@ import ( "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" "github.com/hivecommons/hive/pkg/hivestate" @@ -2279,6 +2280,7 @@ func inClusterHTTPClient(cfg *inClusterConfig) (*http.Client, error) { Timeout: routeExistenceProbeTimeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + DialContext: egressmark.Dialer(0).DialContext, }, }, nil } diff --git a/src/pkg/hub/spoke/self_upgrade.go b/src/pkg/hub/spoke/self_upgrade.go index d699fa2bec..d21d0d23b1 100644 --- a/src/pkg/hub/spoke/self_upgrade.go +++ b/src/pkg/hub/spoke/self_upgrade.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/imageref" ) @@ -686,7 +687,7 @@ func k8sAPIPatchOnce(path string, body []byte) error { client := &http.Client{ Timeout: selfUpgradePatchTimeout, - Transport: &http.Transport{TLSClientConfig: tlsConfig}, + Transport: &http.Transport{TLSClientConfig: tlsConfig, DialContext: egressmark.Dialer(0).DialContext}, } ctx, cancel := context.WithTimeout(context.Background(), selfUpgradePatchTimeout) From 5a590d4d5bf7d03f9bc0e738f5b023450e1e7c96 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 18:36:45 -0700 Subject: [PATCH 043/223] =?UTF-8?q?=F0=9F=90=9B=20hub:=20stop=20flagging?= =?UTF-8?q?=20off-cadence=20and=20scheduled=20agents=20as=20idle-with-work?= =?UTF-8?q?=20(#17825)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit classifyInactiveAgent ignored AgentSummary.ExpectedActive, so an agent with no cadence in the governor's current mode (e.g. redteam while quiet) was alerted as idle whenever the hive had queued work. Spokes also reported KickIntervalSec=0 for times/cron cadences, dropping the hub to the 45m rule between slots. Skip the idle rule when the spoke says the agent is not expected active, and report the gap from the last fired slot to the next as the kick interval for scheduled cadences. Signed-off-by: Andrew Anderson --- ...xed-hub-idle-off-cadence-false-positive.md | 1 + src/pkg/hub/agent_inactivity.go | 7 +++++ src/pkg/hub/agent_inactivity_test.go | 21 ++++++++++++++ src/pkg/hub/spoke/heartbeat_activity.go | 29 ++++++++++++++++++- src/pkg/hub/spoke/heartbeat_activity_test.go | 25 ++++++++++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-hub-idle-off-cadence-false-positive.md diff --git a/changelog.d/fixed-hub-idle-off-cadence-false-positive.md b/changelog.d/fixed-hub-idle-off-cadence-false-positive.md new file mode 100644 index 0000000000..dfd16fe914 --- /dev/null +++ b/changelog.d/fixed-hub-idle-off-cadence-false-positive.md @@ -0,0 +1 @@ +- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. diff --git a/src/pkg/hub/agent_inactivity.go b/src/pkg/hub/agent_inactivity.go index 0cd7fb41e4..3531af4a61 100644 --- a/src/pkg/hub/agent_inactivity.go +++ b/src/pkg/hub/agent_inactivity.go @@ -153,6 +153,13 @@ func classifyInactiveAgent(a AgentSummary, queuedWork int, now time.Time) agentI // turns a healthy fresh digest into "idle with work queued". return agentInactiveNone } + if !legacyAgent(a) && !a.ExpectedActive { + // A new-protocol spoke that says the agent is not expected active in + // the governor's current mode (no cadence, or a paused one — e.g. + // redteam in quiet mode) will never kick it; sitting idle is the + // configured behaviour, not a fault. + return agentInactiveNone + } lastActivity, activityOK := parseAgentTime(a.LastActivityAt) if !activityOK { // The spoke has never observed a pane change for this agent. That is diff --git a/src/pkg/hub/agent_inactivity_test.go b/src/pkg/hub/agent_inactivity_test.go index 39aec10838..a1fbba6c37 100644 --- a/src/pkg/hub/agent_inactivity_test.go +++ b/src/pkg/hub/agent_inactivity_test.go @@ -188,6 +188,27 @@ func TestIdleRuleRespectsKickCadence(t *testing.T) { } } +func TestIdleRuleSkipsOffCadenceAgents(t *testing.T) { + now := time.Now() + a := AgentSummary{ + Name: "redteam", State: "running", Backend: "copilot", + Enabled: true, CanOpenIssue: true, CanOpenPR: true, + StartedAt: settled(now), LastActivityAt: activeAt(now, 6*time.Hour), + } + + // Off-cadence in the governor's current mode (projectbluefin redteam in + // quiet mode): the governor never kicks it, so idleness is by design. + if got := classifyInactiveAgent(a, 19, now); got != agentInactiveNone { + t.Fatalf("off-cadence agent classified %v; want none", got.label()) + } + + // Same agent once the mode gives it a cadence is held to the idle rule. + a.ExpectedActive = true + if got := classifyInactiveAgent(a, 19, now); got != agentInactiveIdleWithWork { + t.Fatalf("expected-active idle agent classified %v; want idle-with-work", got.label()) + } +} + func TestIdleRuleSkipsWriteIncapableAdvisoryAgents(t *testing.T) { now := time.Now() a := AgentSummary{ diff --git a/src/pkg/hub/spoke/heartbeat_activity.go b/src/pkg/hub/spoke/heartbeat_activity.go index 566c4fd5fc..bdcec1e543 100644 --- a/src/pkg/hub/spoke/heartbeat_activity.go +++ b/src/pkg/hub/spoke/heartbeat_activity.go @@ -122,12 +122,39 @@ func HeartbeatKickInterval(govState governor.State, name string, proc *crux.Proc return 0 } cadence, ok := govState.Cadences[name] - if !ok || cadence.Paused || cadence.Interval <= 0 { + if !ok || cadence.Paused { + return 0 + } + if cadence.Schedule != "" && cadence.Schedule.Mode() != config.CadenceModeInterval { + return scheduledKickGap(cadence, govState.LastKick[name], time.Now()) + } + if cadence.Interval <= 0 { return 0 } return cadence.Interval } +// scheduledKickGap is the cadence evidence for a times/cron agent: the gap +// from the slot it last fired at to the slot it fires at next. The hub's +// idle rule compares pane quiet against this (plus slack), so without it a +// "09:00,12:00,15:00 weekdays" agent reads as idle-with-work every evening +// and all weekend — exactly when it is supposed to be quiet. With no fired +// slot yet the gap runs from now to the first slot. Zero means no evidence. +func scheduledKickGap(cadence governor.AgentCadence, lastKick, now time.Time) time.Duration { + base := cadence.LastOccurrence + if base.IsZero() { + base = lastKick + } + if base.IsZero() || base.After(now) { + base = now + } + next, ok := cadence.Schedule.NextAfter(base) + if !ok || !next.After(base) { + return 0 + } + return next.Sub(base) +} + func QuotaExhaustedAgentCount(agents []AgentSummary) int { count := 0 for _, a := range agents { diff --git a/src/pkg/hub/spoke/heartbeat_activity_test.go b/src/pkg/hub/spoke/heartbeat_activity_test.go index fd0e6053cc..eb8100c9a9 100644 --- a/src/pkg/hub/spoke/heartbeat_activity_test.go +++ b/src/pkg/hub/spoke/heartbeat_activity_test.go @@ -102,6 +102,31 @@ func TestHeartbeatKickInterval(t *testing.T) { } } +func TestScheduledKickGapSpansSlots(t *testing.T) { + // Weekday 09:00/12:00/15:00 UTC. Last fired Friday 15:00 → next slot is + // Monday 09:00, so the gap must cover the whole weekend, not 3h. + cad := governor.AgentCadence{Schedule: config.Cadence(`tod:{"times":["09:00","12:00","15:00"],"tz":"UTC","days":["mon","tue","wed","thu","fri"]}`)} + if err := cad.Schedule.Validate(); err != nil { + t.Fatalf("fixture cadence invalid: %v", err) + } + fri := time.Date(2026, 11, 13, 15, 0, 0, 0, time.UTC) + cad.LastOccurrence = fri + if got := scheduledKickGap(cad, fri, fri.Add(5*time.Hour)); got != 66*time.Hour { + t.Errorf("weekend gap = %v, want 66h", got) + } + // Mid-day slot: 09:00 → 12:00. + cad.LastOccurrence = time.Date(2026, 11, 11, 9, 0, 0, 0, time.UTC) + if got := scheduledKickGap(cad, time.Time{}, cad.LastOccurrence.Add(time.Hour)); got != 3*time.Hour { + t.Errorf("intra-day gap = %v, want 3h", got) + } + // Never fired: gap runs from now to the first slot. + cad.LastOccurrence = time.Time{} + now := time.Date(2026, 11, 11, 7, 0, 0, 0, time.UTC) + if got := scheduledKickGap(cad, time.Time{}, now); got != 2*time.Hour { + t.Errorf("unfired gap = %v, want 2h", got) + } +} + func TestQuotaExhaustedCountsAndReasons(t *testing.T) { statuses := map[string]*crux.Process{ "counted": {State: crux.StateRunning, QuotaExhausted: true}, From 10b5bcc1b19bb6700dd7258cff1a39158332156c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:50:48 +0000 Subject: [PATCH 044/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20use=20unique=20te?= =?UTF-8?q?mp=20files=20when=20persisting=20hive=20and=20reach=20state=20(?= =?UTF-8?q?#17740)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use unique temp files when persisting hive and reach state Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: re-run PR verifier after cancelled concurrent run Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: refresh head to clear a Copilot PR Automation run stuck in queued Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-state-persist-rename-race.md | 1 + src/pkg/snapshot/state.go | 20 +++++++++++-- src/pkg/snapshot/state_test.go | 29 +++++++++++++++++++ src/pkg/tracing/reach.go | 20 +++++++++++-- src/pkg/tracing/reach_test.go | 28 ++++++++++++++++++ 5 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-state-persist-rename-race.md diff --git a/changelog.d/fixed-state-persist-rename-race.md b/changelog.d/fixed-state-persist-rename-race.md new file mode 100644 index 0000000000..cbe357487b --- /dev/null +++ b/changelog.d/fixed-state-persist-rename-race.md @@ -0,0 +1 @@ +- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). diff --git a/src/pkg/snapshot/state.go b/src/pkg/snapshot/state.go index fdc7230281..419489f0bc 100644 --- a/src/pkg/snapshot/state.go +++ b/src/pkg/snapshot/state.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "time" "github.com/hivecommons/hive/pkg/config" @@ -183,11 +184,26 @@ func SaveState(path string, state *PersistedState, logger *slog.Logger) error { return fmt.Errorf("marshaling state: %w", err) } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { + // A unique temp name per call: concurrent savers sharing path+".tmp" + // would rename each other's file away and fail with ENOENT (#17726). + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { return fmt.Errorf("writing temp state file: %w", err) } + tmp := f.Name() + _, werr := f.Write(data) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Chmod(tmp, 0o644) + } + if werr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("writing temp state file: %w", werr) + } if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) return fmt.Errorf("renaming state file: %w", err) } diff --git a/src/pkg/snapshot/state_test.go b/src/pkg/snapshot/state_test.go index 2d9905f7e2..61b6de9a78 100644 --- a/src/pkg/snapshot/state_test.go +++ b/src/pkg/snapshot/state_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -239,3 +240,31 @@ func TestLoadState_ReadErrorNotMissing(t *testing.T) { t.Error("state should be nil on read error") } } + +func TestSaveState_ConcurrentWritersDoNotRace(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- SaveState(path, &PersistedState{Agents: map[string]AgentState{}}, testLogger()) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent SaveState: %v", err) + } + } + if _, err := LoadState(path, testLogger()); err != nil { + t.Fatalf("LoadState after concurrent saves: %v", err) + } + leftovers, _ := filepath.Glob(path + "*.tmp") + if len(leftovers) != 0 { + t.Fatalf("leftover temp files: %v", leftovers) + } +} diff --git a/src/pkg/tracing/reach.go b/src/pkg/tracing/reach.go index 78caf7461e..7bdb660b81 100644 --- a/src/pkg/tracing/reach.go +++ b/src/pkg/tracing/reach.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "sort" "sync" "sync/atomic" @@ -388,11 +389,26 @@ func SaveReachState(path string) error { if err != nil { return fmt.Errorf("marshal reach state: %w", err) } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { + // Unique temp name per call so concurrent savers cannot rename each + // other's file away (ENOENT, #17728). + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { return fmt.Errorf("write reach state: %w", err) } + tmp := f.Name() + _, werr := f.Write(data) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Chmod(tmp, 0o644) + } + if werr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("write reach state: %w", werr) + } if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) return fmt.Errorf("rename reach state: %w", err) } return nil diff --git a/src/pkg/tracing/reach_test.go b/src/pkg/tracing/reach_test.go index bb5feb4a31..8a8da61d02 100644 --- a/src/pkg/tracing/reach_test.go +++ b/src/pkg/tracing/reach_test.go @@ -311,3 +311,31 @@ func TestReach_WindowRollover(t *testing.T) { c.winStart.Load(), c.winTotal.Load(), c.winErrors.Load()) } } + +func TestSaveReachState_ConcurrentWritersDoNotRace(t *testing.T) { + resetReach(t) + _, span := StartSpan(context.Background(), "governor.eval_cycle") + span.End() + path := filepath.Join(t.TempDir(), "reach-state.json") + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- SaveReachState(path) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent SaveReachState: %v", err) + } + } + entries, _ := filepath.Glob(path + "*.tmp") + if len(entries) != 0 { + t.Fatalf("leftover temp files: %v", entries) + } +} From da10b788dc8ee5ff50c2b53112cd1a3dad62439f Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:53:24 +0000 Subject: [PATCH 045/223] =?UTF-8?q?=F0=9F=90=9B=20hub=20heartbeat:=20back?= =?UTF-8?q?=20off=20and=20classify=20a=20permanent=20403=20rejection=20(#1?= =?UTF-8?q?7823)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 hub heartbeat: back off and classify a permanent 403 rejection instead of retrying every tick Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 fleetreport: add threshold case for hub_heartbeat_rejected_unprovisioned TestEveryRuntimeClassTriggerAndNonTriggerThresholds requires every class in Classes to have a trigger/non-trigger case; the new class was missing one. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-hub-heartbeat-403-backoff.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/fleetreport/runtime.go | 2 + src/pkg/fleetreport/runtime_test.go | 1 + src/pkg/hub/spoke/heartbeat.go | 34 ++++++++++ src/pkg/hub/spoke/heartbeat_testhooks.go | 1 + .../hub/spoke/heartbeat_unprovisioned_test.go | 63 +++++++++++++++++++ 7 files changed, 103 insertions(+) create mode 100644 changelog.d/fixed-hub-heartbeat-403-backoff.md create mode 100644 src/pkg/hub/spoke/heartbeat_unprovisioned_test.go diff --git a/changelog.d/fixed-hub-heartbeat-403-backoff.md b/changelog.d/fixed-hub-heartbeat-403-backoff.md new file mode 100644 index 0000000000..cf59938b9a --- /dev/null +++ b/changelog.d/fixed-hub-heartbeat-403-backoff.md @@ -0,0 +1 @@ +- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index bd96ad2143..ad7f77ae2a 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -145,6 +145,7 @@ title is `[selfreport] : `. | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | +| `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index c0b7a6779d..9e9b3c3c48 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -77,6 +77,7 @@ const ( ClassBeadFactIngestFailed = "bead_fact_ingest_failed" ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" ClassInferenceUpstreamError = "inference_upstream_error" @@ -197,6 +198,7 @@ var Classes = map[string]ClassSpec{ ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, + ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 1593918264..da3b933e4a 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -453,6 +453,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, + {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, {class: ClassProxyPlainForwardRefused, message: "proxy: plain HTTP forward refused — destination is not globally routable"}, diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 5ab3c0990e..86c71807b5 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1528,11 +1528,38 @@ func StartHeartbeat(ctx context.Context, hubURL string, collect StatusCollector, logger.Info("hub heartbeat stopped") return case <-ticker.C: + if heartbeatUnprovisionedBackoffActive(time.Now()) { + // Keep the attempt clock advancing so /api/livez does not + // read the deliberate pause as a stalled loop. + recordHeartbeatAttempt() + continue + } processHeartbeatResponse(sendHeartbeat(ctx, hubURL, collect, logger, freshCollect)) } } } +// heartbeatUnprovisionedBackoff is how long the loop pauses after the hub +// answers 403 "hosted/saas hive IDs can only be created via provisioning". +// The hub never auto-creates hosted IDs, so retrying every tick cannot +// succeed until an operator provisions (or the hive is re-IDed); a slow probe +// still notices when that happens. A var so tests can shorten it. +var heartbeatUnprovisionedBackoff = 10 * time.Minute + +// heartbeatUnprovisionedUntil is the UnixNano deadline of that pause (0 = none). +var heartbeatUnprovisionedUntil atomic.Int64 + +func heartbeatUnprovisionedBackoffActive(now time.Time) bool { + until := heartbeatUnprovisionedUntil.Load() + return until != 0 && now.UnixNano() < until +} + +// hubHeartbeatRejectsUnprovisioned reports whether the hub refused the beat +// because it does not know this hosted/saas hive ID (a permanent condition). +func hubHeartbeatRejectsUnprovisioned(status int, body string) bool { + return status == http.StatusForbidden && strings.Contains(body, "can only be created via provisioning") +} + // waitForReady's knobs are vars (not consts) for the same reason as // taskPushInterval: tests need a StartHeartbeat that reaches its send loop in // milliseconds, not after a 3-minute readiness wait. Production keeps the @@ -1843,6 +1870,12 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa if resp.StatusCode >= 300 { respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + if hubHeartbeatRejectsUnprovisioned(resp.StatusCode, string(respBody)) { + heartbeatUnprovisionedUntil.Store(time.Now().Add(heartbeatUnprovisionedBackoff).UnixNano()) + logger.Warn("hub heartbeat rejected: hive id is not provisioned on the hub — backing off", + "status", resp.StatusCode, "body", string(respBody), "retry_in", heartbeatUnprovisionedBackoff.String()) + return nil + } logger.Warn("hub heartbeat rejected", "status", resp.StatusCode, "body", string(respBody)) if hubHeartbeatStatusIsDefect(resp.StatusCode) { fleetreport.Observe(fleetreport.ClassUnexpectedHub5xx, "heartbeat", fmt.Sprintf("hub heartbeat returned HTTP %d", resp.StatusCode), string(respBody)) @@ -1850,6 +1883,7 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa return nil } fleetreport.Clear(fleetreport.ClassUnexpectedHub5xx, "heartbeat") + heartbeatUnprovisionedUntil.Store(0) // The hub accepted the beat (status < 300) — this is what /api/livez // treats as "the heartbeat goroutine is alive and doing its job", so diff --git a/src/pkg/hub/spoke/heartbeat_testhooks.go b/src/pkg/hub/spoke/heartbeat_testhooks.go index 87216c3f17..48743a0619 100644 --- a/src/pkg/hub/spoke/heartbeat_testhooks.go +++ b/src/pkg/hub/spoke/heartbeat_testhooks.go @@ -29,6 +29,7 @@ func SetHeartbeatStateForTest(enabled bool, lastAttempt, lastSuccess time.Time) // still had fresh stats to fall back on. func ResetHeartbeatStateForTest() { SetHeartbeatStateForTest(false, time.Time{}, time.Time{}) + heartbeatUnprovisionedUntil.Store(0) lastGoodPayload.Lock() // Invalidate any collect goroutine still running from an earlier test // (same lock as its late store) so it cannot repopulate the cache. diff --git a/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go b/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go new file mode 100644 index 0000000000..4aa9227c4d --- /dev/null +++ b/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go @@ -0,0 +1,63 @@ +package spoke + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/fleetreport" +) + +func TestPostHeartbeat_UnprovisionedForbiddenBacksOff(t *testing.T) { + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "hosted/saas hive IDs can only be created via provisioning", http.StatusForbidden) + })) + defer srv.Close() + + if resp := postHeartbeatToHub(context.Background(), srv.URL, &HeartbeatPayload{}, nil2Logger()); resp != nil { + t.Fatalf("rejected beat returned a response: %+v", resp) + } + if !heartbeatUnprovisionedBackoffActive(time.Now()) { + t.Fatal("expected backoff after unprovisioned 403") + } + if heartbeatUnprovisionedBackoffActive(time.Now().Add(heartbeatUnprovisionedBackoff + time.Second)) { + t.Fatal("backoff must expire") + } +} + +func TestPostHeartbeat_OtherForbiddenDoesNotBackOff(t *testing.T) { + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "bad secret", http.StatusForbidden) + })) + defer srv.Close() + + postHeartbeatToHub(context.Background(), srv.URL, &HeartbeatPayload{}, nil2Logger()) + if heartbeatUnprovisionedBackoffActive(time.Now()) { + t.Fatal("a 403 without the provisioning body must not back off") + } +} + +func TestHubHeartbeatRejectsUnprovisioned(t *testing.T) { + body := "hosted/saas hive IDs can only be created via provisioning" + if !hubHeartbeatRejectsUnprovisioned(http.StatusForbidden, body) { + t.Error("403 + body should match") + } + if hubHeartbeatRejectsUnprovisioned(http.StatusUnauthorized, body) { + t.Error("401 must not match") + } +} + +func TestUnprovisionedWarnIsLogFedClass(t *testing.T) { + msg := "hub heartbeat rejected: hive id is not provisioned on the hub — backing off" + if got := fleetreport.LogFedClass(msg); got != fleetreport.ClassHubHeartbeatUnprovisioned { + t.Fatalf("LogFedClass = %q", got) + } +} From 3239727844a2a8145babf4db8f1784813360b415 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 02:05:31 +0000 Subject: [PATCH 046/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.227.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.227.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 16 ++++++++++++++++ .../added-terminal-pane-substrate-kicks.md | 1 - .../fixed-hub-idle-off-cadence-false-positive.md | 1 - changelog.d/fixed-k8s-api-client-egress-mark.md | 1 - .../security-marketplace-refresh-corpus-only.md | 1 - changelog.d/security-substrate-unpack-symlink.md | 1 - 6 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-terminal-pane-substrate-kicks.md delete mode 100644 changelog.d/fixed-hub-idle-off-cadence-false-positive.md delete mode 100644 changelog.d/fixed-k8s-api-client-egress-mark.md delete mode 100644 changelog.d/security-marketplace-refresh-corpus-only.md delete mode 100644 changelog.d/security-substrate-unpack-symlink.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4da2380372..1481511f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.227.0) + +### Added + +- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. + +### Fixed + +- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. +- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). + +### Security + +- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. +- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. + ## 2026-11-12 (v5.226.0) ### Added diff --git a/changelog.d/added-terminal-pane-substrate-kicks.md b/changelog.d/added-terminal-pane-substrate-kicks.md deleted file mode 100644 index 1514ab649f..0000000000 --- a/changelog.d/added-terminal-pane-substrate-kicks.md +++ /dev/null @@ -1 +0,0 @@ -- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. diff --git a/changelog.d/fixed-hub-idle-off-cadence-false-positive.md b/changelog.d/fixed-hub-idle-off-cadence-false-positive.md deleted file mode 100644 index dfd16fe914..0000000000 --- a/changelog.d/fixed-hub-idle-off-cadence-false-positive.md +++ /dev/null @@ -1 +0,0 @@ -- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. diff --git a/changelog.d/fixed-k8s-api-client-egress-mark.md b/changelog.d/fixed-k8s-api-client-egress-mark.md deleted file mode 100644 index b33246cd5f..0000000000 --- a/changelog.d/fixed-k8s-api-client-egress-mark.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). diff --git a/changelog.d/security-marketplace-refresh-corpus-only.md b/changelog.d/security-marketplace-refresh-corpus-only.md deleted file mode 100644 index 9c888537a7..0000000000 --- a/changelog.d/security-marketplace-refresh-corpus-only.md +++ /dev/null @@ -1 +0,0 @@ -- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. diff --git a/changelog.d/security-substrate-unpack-symlink.md b/changelog.d/security-substrate-unpack-symlink.md deleted file mode 100644 index 28e56900e6..0000000000 --- a/changelog.d/security-substrate-unpack-symlink.md +++ /dev/null @@ -1 +0,0 @@ -- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. From 52c64f9c338c513c940a82f99b4fc2addc0867da Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 19:19:13 -0700 Subject: [PATCH 047/223] =?UTF-8?q?=E2=9C=A8=20knowledge:=20Plaud=20voice?= =?UTF-8?q?=20notes=20as=20a=20knowledge=20source=20(#17821)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: Plaud voice notes as a knowledge source Add a plaud knowledge connector that reads Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server over stdio, with a minimal JSON-RPC MCP client, Connect/Disconnect sign-in on the Knowledge Integrations card, docs and tests. Fixes #17820 Signed-off-by: Andrew Anderson * 🌱 knowledge/plaud: port dashboard edits to static-src, document auth routes in openapi.json, drop unused field Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-knowledge-plaud-source.md | 1 + dashboard/openapi.json | 373 ++++++++++++++++ src/docs/api-reference/knowledge.md | 3 + src/docs/api-reference/packs-and-acmm.md | 2 +- src/docs/knowledge-connectors.md | 89 +++- src/pkg/dashboard/api.go | 1 + .../dashboard/api_knowledge_connector_auth.go | 114 +++++ .../api_knowledge_connector_auth_test.go | 81 ++++ .../knowledge_integrations_modal_test.go | 42 ++ .../dashboard/static-src/js/config-state.js | 1 + .../dashboard/static-src/js/config-tabs.js | 5 +- src/pkg/dashboard/static-src/js/knowledge.js | 89 ++++ src/pkg/dashboard/static/index.html | 95 +++- src/pkg/knowledge/connector/connector.go | 3 +- src/pkg/knowledge/connector/connector_test.go | 2 +- src/pkg/knowledge/connector/mcpstdio.go | 379 ++++++++++++++++ src/pkg/knowledge/connector/mcpstdio_test.go | 182 ++++++++ src/pkg/knowledge/connector/plaud.go | 421 ++++++++++++++++++ src/pkg/knowledge/connector/plaud_auth.go | 374 ++++++++++++++++ src/pkg/knowledge/connector/plaud_test.go | 229 ++++++++++ src/pkg/knowledge/connector/syncer.go | 10 + 21 files changed, 2489 insertions(+), 7 deletions(-) create mode 100644 changelog.d/added-knowledge-plaud-source.md create mode 100644 src/pkg/dashboard/api_knowledge_connector_auth.go create mode 100644 src/pkg/dashboard/api_knowledge_connector_auth_test.go create mode 100644 src/pkg/knowledge/connector/mcpstdio.go create mode 100644 src/pkg/knowledge/connector/mcpstdio_test.go create mode 100644 src/pkg/knowledge/connector/plaud.go create mode 100644 src/pkg/knowledge/connector/plaud_auth.go create mode 100644 src/pkg/knowledge/connector/plaud_test.go diff --git a/changelog.d/added-knowledge-plaud-source.md b/changelog.d/added-knowledge-plaud-source.md new file mode 100644 index 0000000000..c7cf5b02d6 --- /dev/null +++ b/changelog.d/added-knowledge-plaud-source.md @@ -0,0 +1 @@ +- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 72944b0d0f..cc87eca843 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -12676,6 +12676,379 @@ } } }, + "/api/knowledge/connectors/{name}/auth": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Get a knowledge connector's sign-in state", + "description": "Returns the cached sign-in state of a connector that authenticates interactively (e.g. `plaud` via the Plaud MCP OAuth flow). Pass `refresh=1` to re-query the upstream account. Tokens are never returned; they stay in the connector's own credential store.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + }, + { + "name": "refresh", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "1", + "true" + ] + }, + "description": "Re-query the upstream account instead of returning the cached state" + } + ], + "responses": { + "200": { + "description": "Sign-in state", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/connect": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Start a knowledge connector's interactive sign-in", + "description": "Owner-only. Starts the connector's OAuth login (for `plaud`, the MCP server's `login` tool) and returns the pending state including `auth_url` for the user to open in a browser. The login session is kept alive for the callback; poll `GET /api/knowledge/connectors/{name}/auth?refresh=1` for the outcome.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "responses": { + "202": { + "description": "Login started (or already connected)", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Owner access required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "The connector's MCP server failed to start the login", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/disconnect": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Sign a knowledge connector out", + "description": "Owner-only. Signs the connector out upstream (for `plaud`, the MCP server's `logout` tool), which removes the token from the connector's own credential store, and returns the new state.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "responses": { + "200": { + "description": "Signed out", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Owner access required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "The connector's MCP server failed to sign out", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/api/knowledge/connectors/{name}/sync": { "post": { "tags": [ diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index ac9bc8f73a..1228e273fb 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -62,3 +62,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | +| `GET` | `/api/knowledge/connectors/{name}/auth` | Dashboard auth/session | Interactive sign-in state for a connector (Plaud OAuth): connected, account, pending, auth_url; `?refresh=1` re-checks upstream (400 for connector types without interactive sign-in) | `pkg/dashboard/api_knowledge_connector_auth.go:25` | +| `POST` | `/api/knowledge/connectors/{name}/auth/connect` | Owner only | Start the connector's sign-in flow (202 with state incl. `auth_url`) | `pkg/dashboard/api_knowledge_connector_auth.go:26` | +| `POST` | `/api/knowledge/connectors/{name}/auth/disconnect` | Owner only | Sign out and revoke the connector's credential upstream | `pkg/dashboard/api_knowledge_connector_auth.go:27` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index d0067822c6..f01162b768 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:555`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:556`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index 69f7c3b801..d9328667f9 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -1,6 +1,6 @@ # Knowledge connectors -External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents and GitHub wikis) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. +External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis and Plaud voice notes) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. For the shared configuration fields, the fact format and how connectors relate to `git_sources` and `documents`, see [knowledge-curator.md → Connectors](knowledge-curator.md#connectors-knowledgeconnectors). @@ -10,7 +10,7 @@ For the shared configuration fields, the fact format and how connectors relate t knowledge: connectors: - name: team-wiki # lowercase letters, digits, dashes - type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint + type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud enabled: true interval: 15m # minimum 1m; blank = 15m layer: project # personal | project | org | community @@ -218,6 +218,91 @@ knowledge: ends the content when any were dropped. See [Context7 content cap](knowledge-curator.md#context7-content-cap). +## Plaud Note Pro (voice notes, `type: plaud`) + +The Plaud connector turns the recordings on a [Plaud Note / Note Pro](https://plaud.ai) +into knowledge facts: one fact per recording with the recording time, +duration, Plaud's AI summary and the full speaker-labelled transcript. Agents +search and cite them like any other fact, so a voice memo recorded on the way +to work can inform the issue or PR an agent picks up an hour later. + +It talks to the **official, free Plaud MCP server** (`npx -y @plaud-ai/mcp`) +over stdio as an MCP client — there is no Zapier/Make/IFTTT hop and nothing to +pay beyond your existing Plaud plan. Plaud lists the MCP server as a Preview +feature on every tier including Starter; Plaud's docs note that preview +features may later move behind paid plans. + +```yaml +knowledge: + connectors: + - name: plaud + type: plaud + layer: personal + interval: 15m + scope: + command: npx -y @plaud-ai/mcp@latest # default; any stdio MCP command + since_days: "30" # how far back the first sync reaches + max_files: "200" # recordings fetched per sync + include_audio: "false" # add Plaud's 24h presigned MP3 link + query: "" # optional name filter (list_files query) + tags: "voice, ideas" # extra tags written as attr_tags + # home: /data/knowledge/connector-state/plaud/home # token store; default shown, or "inherit" +``` + +No `auth` block: Plaud signs you in with its own OAuth flow and keeps the token +in the MCP server's store (`$HOME/.plaud/tokens-mcp.json`). hive never sees, +stores or logs the token. By default the connector points the MCP server's +`HOME` at its own state directory under `/data/knowledge/connector-state//home` +so the token survives restarts; set `home: inherit` to use the hive process's +HOME instead. + +### Setup + +1. The hive host (or container image) needs **Node.js ≥ 20** so `npx` can run + the Plaud MCP server. +2. Add the connector from **Knowledge → 🔌 Integrations → Plaud voice notes** + (or the YAML above) and restart the hive so the syncer picks it up. +3. On the Plaud card click **Connect Plaud**. The card shows the Plaud + authorization URL; open it, sign in and click **Authorize**. The OAuth + callback lands on `localhost:8199` of the machine running hive, so for a + remote hive forward that port first (`ssh -L 8199:localhost:8199 user@host`) + and open the URL from the forwarding machine. The login session stays open + for 10 minutes; click **Check** to confirm "connected as …". + - Alternative when port-forwarding is impractical: authorize once on your + laptop with `npx -y @plaud-ai/mcp@latest install`, then copy + `~/.plaud/tokens-mcp.json` into the connector's `home` directory + (`…/connector-state//home/.plaud/`). The file is Plaud's own + token store, not a hive secret. +4. Click **Sync now**. New recordings are picked up every `interval`. + +### Behaviour + +- Each sync calls `list_files` (from the newest recording seen minus a 24 h + overlap, or `since_days` back on the first run), then `get_note` and + `get_transcript` for each recording. A recording whose transcript and + summary are both still processing is skipped and retried next sync; the + status reports it as a partial sync. +- Facts carry `source: plaud`, `attr_plaud_file_id`, `attr_recorded_at`, + `attr_duration_ms` and `attr_device_serial`. Audio is never downloaded; + with `include_audio: true` the fact links Plaud's 24-hour presigned URL. +- The listing is incremental, so recordings deleted in Plaud are not + tombstoned automatically; remove the fact file or the connector. +- Sign-in state (`connected as`, pending authorization, last error) comes from + `GET /api/knowledge/connectors/{name}/auth`; see [API](#api). + +### Privacy and cost + +- Transcripts and summaries are produced in **Plaud's cloud** and fetched from + there; the hive stores them in the vault for the layer you choose (use + `personal` unless the whole hive should see them). Do not record material + that may not leave your organisation. +- **Local-only alternative:** Plaud's app can auto-sync notes to an Obsidian + vault or Drive/Dropbox folder, and exports TXT/Markdown/DOCX by hand. Point + the existing `git` connector at a Git-synced copy of that folder (see the + Obsidian entry in 🔌 Integrations), or import single files with a + `document` connector (`file_path`). No code, still $0, and no MCP sign-in. +- Everything in this section costs **$0** beyond your Plaud subscription. + ## Notion (`type: notion`) The Notion connector syncs pages and database rows through the public Notion diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index 52c63d7fa0..af8380cc4a 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -548,6 +548,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { s.mux.HandleFunc("GET /api/hotspots", s.handleHotspots) // Knowledge connector status + manual sync (#11069, api_knowledge_connectors.go). s.registerKnowledgeConnectorRoutes() + s.registerKnowledgeConnectorAuthRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) // Per-agent Variables tab (#11458, api_variables.go). s.registerAgentVariableRoutes() diff --git a/src/pkg/dashboard/api_knowledge_connector_auth.go b/src/pkg/dashboard/api_knowledge_connector_auth.go new file mode 100644 index 0000000000..a5bd882be1 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connector_auth.go @@ -0,0 +1,114 @@ +package dashboard + +import ( + "context" + "net/http" + "time" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// knowledgeConnectorLookup is implemented by the connector syncer so the +// dashboard can reach a connector's optional interfaces (Authenticator). +type knowledgeConnectorLookup interface { + Connector(name string) (connector.Connector, bool) +} + +// knowledgeAuthTimeout bounds one auth status/connect/disconnect request, +// which may spawn the connector's MCP server. +const knowledgeAuthTimeout = 90 * time.Second + +// Interactive connector sign-in (Plaud OAuth through its MCP server): status, +// connect (returns the authorization URL) and disconnect. Tokens stay in the +// vendor tool's own store and are never returned. +func (s *Server) registerKnowledgeConnectorAuthRoutes() { + s.mux.HandleFunc("GET /api/knowledge/connectors/{name}/auth", s.handleKnowledgeConnectorAuthStatus) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/connect", s.handleKnowledgeConnectorAuthConnect) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/disconnect", s.handleKnowledgeConnectorAuthDisconnect) +} + +// knowledgeAuthenticator resolves the named connector's Authenticator, +// writing the matching error response when it is unavailable. +func (s *Server) knowledgeAuthenticator(w http.ResponseWriter, name string) (connector.Authenticator, bool) { + rt := s.knowledgeConnectors() + if rt == nil { + jsonError(w, "no knowledge connectors are configured", http.StatusNotFound) + return nil, false + } + lookup, ok := rt.(knowledgeConnectorLookup) + if !ok { + jsonError(w, "connector sign-in is not available on this runtime", http.StatusServiceUnavailable) + return nil, false + } + c, ok := lookup.Connector(name) + if !ok { + jsonError(w, "unknown knowledge connector", http.StatusNotFound) + return nil, false + } + auth, ok := c.(connector.Authenticator) + if !ok { + jsonError(w, "this connector type does not use interactive sign-in", http.StatusBadRequest) + return nil, false + } + return auth, true +} + +func knowledgeAuthContext(r *http.Request) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(r.Context()), knowledgeAuthTimeout) +} + +// handleKnowledgeConnectorAuthStatus returns the cached sign-in state, or a +// fresh one when ?refresh=1 (which may start the MCP server briefly). +func (s *Server) handleKnowledgeConnectorAuthStatus(w http.ResponseWriter, r *http.Request) { + auth, ok := s.knowledgeAuthenticator(w, r.PathValue("name")) + if !ok { + return + } + refresh := r.URL.Query().Get("refresh") == "1" + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + jsonResponse(w, auth.AuthStatus(ctx, refresh)) +} + +// handleKnowledgeConnectorAuthConnect (owner) starts the sign-in flow and +// answers 202 with the state; auth_url is the page the owner must open. +func (s *Server) handleKnowledgeConnectorAuthConnect(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := auth.Connect(ctx) + if err != nil { + jsonError(w, "connect failed: "+err.Error(), http.StatusBadGateway) + return + } + s.auditFromRequest(r, "knowledge_connector_connect", auditDetail("connector", name), "") + jsonStatusResponse(w, http.StatusAccepted, st) +} + +// handleKnowledgeConnectorAuthDisconnect (owner) signs out upstream. +func (s *Server) handleKnowledgeConnectorAuthDisconnect(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := auth.Disconnect(ctx) + if err != nil { + jsonError(w, "disconnect failed: "+err.Error(), http.StatusBadGateway) + return + } + s.auditFromRequest(r, "knowledge_connector_disconnect", auditDetail("connector", name), "") + jsonResponse(w, st) +} diff --git a/src/pkg/dashboard/api_knowledge_connector_auth_test.go b/src/pkg/dashboard/api_knowledge_connector_auth_test.go new file mode 100644 index 0000000000..7a7b7239f2 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connector_auth_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// authConn is a fake connector implementing connector.Authenticator. +type authConn struct { + blockingConn + state connector.AuthState +} + +func (c *authConn) AuthStatus(context.Context, bool) connector.AuthState { return c.state } +func (c *authConn) Connect(context.Context) (connector.AuthState, error) { + c.state = connector.AuthState{Pending: true, AuthURL: "https://auth.example/authorize"} + return c.state, nil +} +func (c *authConn) Disconnect(context.Context) (connector.AuthState, error) { + c.state = connector.AuthState{Message: "signed out"} + return c.state, nil +} + +func decodeAuthState(t *testing.T, body []byte) connector.AuthState { + t.Helper() + var st connector.AuthState + if err := json.Unmarshal(body, &st); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return st +} + +func TestKnowledgeConnectorAuthEndpoints(t *testing.T) { + s, deps := apiServer(t) + if rec := doGet(s, "/api/knowledge/connectors/wiki/auth"); rec.Code != http.StatusNotFound { + t.Fatalf("auth without syncer = %d, want 404", rec.Code) + } + + conn := &authConn{state: connector.AuthState{Connected: true, Account: "andy@example.com"}} + deps.KnowledgeConnectors = newTestConnectorSyncer(t, conn) + + rec := doGet(s, "/api/knowledge/connectors/wiki/auth") + if rec.Code != http.StatusOK { + t.Fatalf("auth status = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("state = %+v", st) + } + if rec := doGet(s, "/api/knowledge/connectors/nope/auth"); rec.Code != http.StatusNotFound { + t.Fatalf("unknown connector = %d, want 404", rec.Code) + } + if rec := doPostNoOwner(s, "/api/knowledge/connectors/wiki/auth/connect", nil); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner connect = %d, want 403", rec.Code) + } + rec = doPost(s, "/api/knowledge/connectors/wiki/auth/connect", nil) + if rec.Code != http.StatusAccepted { + t.Fatalf("connect = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Pending || st.AuthURL != "https://auth.example/authorize" { + t.Fatalf("connect state = %+v", st) + } + rec = doPost(s, "/api/knowledge/connectors/wiki/auth/disconnect", nil) + if rec.Code != http.StatusOK { + t.Fatalf("disconnect = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); st.Connected || st.Message != "signed out" { + t.Fatalf("disconnect state = %+v", st) + } +} + +func TestKnowledgeConnectorAuthNotInteractive(t *testing.T) { + s, deps := apiServer(t) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &blockingConn{}) + if rec := doGet(s, "/api/knowledge/connectors/wiki/auth"); rec.Code != http.StatusBadRequest { + t.Fatalf("non-authenticator connector = %d, want 400: %s", rec.Code, rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 574593bc45..0e79b35a80 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -142,3 +142,45 @@ func TestKnowledgeIntegrationsModalContext7Card(t *testing.T) { } } } + +// TestKnowledgeIntegrationsModalPlaudCard guards the Plaud entry: the card +// exposes Connect/Disconnect/Check wired to the connector auth endpoints, +// the Knowledge pane links to it, and no browser-native dialog is used. +func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { + b, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatalf("reading embedded static/index.html: %v", err) + } + html := string(b) + for _, want := range []string{ + `'plaud': { label: 'Plaud voice notes'`, + `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, + `if (type === 'plaud') html += kbPlaudAuthLine(c.name);`, + `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, + `data-action="kbPlaudConnect"`, + `data-action="kbPlaudDisconnect"`, + `data-action="kbPlaudRefreshAuth"`, + `data-action="kbOpenPlaudIntegration"`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect'`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect'`, + `async function kbPlaudConnect(name)`, + `async function kbPlaudDisconnect(name)`, + `async function kbPlaudRefreshAuth(name)`, + `function kbOpenPlaudIntegration()`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Plaud card snippet %q", want) + } + } + start := strings.Index(html, "const _kbPlaudAuth = {};") + end := strings.Index(html, "function kbOpenPlaudIntegration()") + if start < 0 || end < start { + t.Fatal("Plaud sign-in block not found") + } + block := html[start:end] + for _, native := range []string{"window.confirm(", "window.alert(", " confirm(", " alert(", "prompt("} { + if strings.Contains(block, native) { + t.Errorf("Plaud sign-in block uses a browser-native dialog %q", native) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 6bb1f4d866..459c8a898a 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -127,6 +127,7 @@ { tab: 'Thresholds', section: 'Thresholds', label: 'Delivery SLA targets', key: 'sla.targets', text: 'delivery sla targets sla.targets.start_to_pr_h sla.targets.pr_to_merge_h sla.targets.open_to_merge_h hours sparkline overview stale in-progress' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index c16d315d5b..fbd7bfa2ef 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -289,7 +289,7 @@ } // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -298,6 +298,7 @@ 'github-wiki': { label: 'GitHub Wiki', description: 'Clone and index one or more GitHub wiki repositories.' }, 'google-drive': { label: 'Google Drive', description: 'Sync Google Docs, Sheets, markdown, and text files from Drive folders or shared drives.' }, 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, + 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } }; @@ -313,6 +314,7 @@ // Generic glyphs (no vendor): document sheet and open book, drawn in-house. 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', + 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -328,6 +330,7 @@ 'github-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'master']], 'google-drive': [['folder_ids', 'Folder IDs', 'folder123, folder456'], ['shared_drive_ids', 'Shared drive IDs', 'drive123'], ['include_mime', 'Include MIME groups', 'docs,md,txt']], 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], + 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] }; diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 94a3fdc231..eb322b6c55 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -1975,6 +1975,12 @@ html += ''; html += '
'; + html += '
🎙️ Plaud voice notes'; + html += '
'; + html += '
Make your Plaud Note recordings (Plaud summary + transcript) searchable knowledge via the free official Plaud MCP server. $0 beyond your Plaud plan; sign-in is Plaud OAuth in your browser.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -2421,9 +2427,11 @@ html += '
' + escapeHtml(c.name) + ' ' + escapeHtml(status) + '
'; html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; + if (type === 'plaud') html += kbPlaudAuthLine(c.name); html += '
'; if (owner) { html += '
'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name); html += ''; html += ''; html += ''; @@ -2460,6 +2468,7 @@ html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; html += '
'; @@ -2543,3 +2552,83 @@ await kcSyncNow(name); setTimeout(() => kbLoadIntegrationsModal(_kbIntegrationSelected), 1500); } + + // ── Plaud sign-in (OAuth through the Plaud MCP server) ── + // Cached per connector name; loaded lazily when the Plaud card renders. + const _kbPlaudAuth = {}; + function kbPlaudAuthLine(name) { + const a = _kbPlaudAuth[name]; + if (!a) { + setTimeout(() => kbPlaudAuthLoad(name, false), 0); + return '
Plaud: checking sign-in…
'; + } + let text; + if (a.error) text = 'Plaud: ' + a.error; + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — open the Plaud authorization page ↗ (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as ' + escapeHtml(a.account) + '' : ''); + else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); + return '
' + text + '
'; + } + function kbPlaudAuthButtons(name) { + const a = _kbPlaudAuth[name] || {}; + const n = escapeHtml(name); + let html = ''; + if (a.connected) html += ''; + else html += ''; + html += ''; + return html; + } + async function kbPlaudAuthLoad(name, refresh) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth' + (refresh ? '?refresh=1' : '')); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + } catch (e) { + _kbPlaudAuth[name] = { error: e.message }; + } + if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + } + async function kbPlaudRefreshAuth(name) { + showToast('Checking Plaud sign-in…', 'info'); + await kbPlaudAuthLoad(name, true); + } + async function kbPlaudConnect(name) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + if (data.connected) showToast('Plaud connected' + (data.account ? ' as ' + data.account : ''), 'success'); + else if (data.auth_url) { showToast('Open the Plaud authorization link shown on the card, then click Check.', 'info'); kbPlaudPollAuth(name, 0); } + else showToast('Plaud login started; the authorization URL will appear on the card shortly.', 'info'); + if (!data.connected && !data.auth_url) kbPlaudPollAuth(name, 0); + } catch (e) { + showToast('Plaud connect failed: ' + e.message, 'error'); + } + } + // Poll the cached state while a login is pending (URL arrival, completion). + function kbPlaudPollAuth(name, attempt) { + if (attempt > 60) return; + setTimeout(async () => { + await kbPlaudAuthLoad(name, false); + const a = _kbPlaudAuth[name] || {}; + if (a.connected) { showToast('Plaud connected' + (a.account ? ' as ' + a.account : ''), 'success'); return; } + if (a.pending) kbPlaudPollAuth(name, attempt + 1); + }, 5000); + } + async function kbPlaudDisconnect(name) { + if (!(await hiveConfirm('Sign out of Plaud? Synced recordings stay in the vault until you remove the connector.'))) return; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast('Plaud disconnected', 'success'); + } catch (e) { + showToast('Plaud disconnect failed: ' + e.message, 'error'); + } + } + function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 21292a873d..1511394525 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -45491,6 +45491,12 @@

Import'; html += '

'; + html += '
🎙️ Plaud voice notes'; + html += '
'; + html += '
Make your Plaud Note recordings (Plaud summary + transcript) searchable knowledge via the free official Plaud MCP server. $0 beyond your Plaud plan; sign-in is Plaud OAuth in your browser.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -45937,9 +45943,11 @@ html += '
' + escapeHtml(c.name) + ' ' + escapeHtml(status) + '
'; html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; + if (type === 'plaud') html += kbPlaudAuthLine(c.name); html += '
'; if (owner) { html += '
'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name); html += ''; html += ''; html += ''; @@ -45976,6 +45984,7 @@ html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; html += '
'; @@ -46059,6 +46068,86 @@ await kcSyncNow(name); setTimeout(() => kbLoadIntegrationsModal(_kbIntegrationSelected), 1500); } + + // ── Plaud sign-in (OAuth through the Plaud MCP server) ── + // Cached per connector name; loaded lazily when the Plaud card renders. + const _kbPlaudAuth = {}; + function kbPlaudAuthLine(name) { + const a = _kbPlaudAuth[name]; + if (!a) { + setTimeout(() => kbPlaudAuthLoad(name, false), 0); + return '
Plaud: checking sign-in…
'; + } + let text; + if (a.error) text = 'Plaud: ' + a.error; + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — open the Plaud authorization page ↗ (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as ' + escapeHtml(a.account) + '' : ''); + else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); + return '
' + text + '
'; + } + function kbPlaudAuthButtons(name) { + const a = _kbPlaudAuth[name] || {}; + const n = escapeHtml(name); + let html = ''; + if (a.connected) html += ''; + else html += ''; + html += ''; + return html; + } + async function kbPlaudAuthLoad(name, refresh) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth' + (refresh ? '?refresh=1' : '')); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + } catch (e) { + _kbPlaudAuth[name] = { error: e.message }; + } + if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + } + async function kbPlaudRefreshAuth(name) { + showToast('Checking Plaud sign-in…', 'info'); + await kbPlaudAuthLoad(name, true); + } + async function kbPlaudConnect(name) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + if (data.connected) showToast('Plaud connected' + (data.account ? ' as ' + data.account : ''), 'success'); + else if (data.auth_url) { showToast('Open the Plaud authorization link shown on the card, then click Check.', 'info'); kbPlaudPollAuth(name, 0); } + else showToast('Plaud login started; the authorization URL will appear on the card shortly.', 'info'); + if (!data.connected && !data.auth_url) kbPlaudPollAuth(name, 0); + } catch (e) { + showToast('Plaud connect failed: ' + e.message, 'error'); + } + } + // Poll the cached state while a login is pending (URL arrival, completion). + function kbPlaudPollAuth(name, attempt) { + if (attempt > 60) return; + setTimeout(async () => { + await kbPlaudAuthLoad(name, false); + const a = _kbPlaudAuth[name] || {}; + if (a.connected) { showToast('Plaud connected' + (a.account ? ' as ' + a.account : ''), 'success'); return; } + if (a.pending) kbPlaudPollAuth(name, attempt + 1); + }, 5000); + } + async function kbPlaudDisconnect(name) { + if (!(await hiveConfirm('Sign out of Plaud? Synced recordings stay in the vault until you remove the connector.'))) return; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast('Plaud disconnected', 'success'); + } catch (e) { + showToast('Plaud disconnect failed: ' + e.message, 'error'); + } + } + function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } // ── Vault Management ── function kbOpenVaults() { @@ -54905,6 +54994,7 @@

Configure LiteLLM

{ tab: 'Thresholds', section: 'Thresholds', label: 'Delivery SLA targets', key: 'sla.targets', text: 'delivery sla targets sla.targets.start_to_pr_h sla.targets.pr_to_merge_h sla.targets.open_to_merge_h hours sparkline overview stale in-progress' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; @@ -56185,7 +56275,7 @@

Configure LiteLLM

} // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -56194,6 +56284,7 @@

Configure LiteLLM

'github-wiki': { label: 'GitHub Wiki', description: 'Clone and index one or more GitHub wiki repositories.' }, 'google-drive': { label: 'Google Drive', description: 'Sync Google Docs, Sheets, markdown, and text files from Drive folders or shared drives.' }, 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, + 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } }; @@ -56209,6 +56300,7 @@

Configure LiteLLM

// Generic glyphs (no vendor): document sheet and open book, drawn in-house. 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', + 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -56224,6 +56316,7 @@

Configure LiteLLM

'github-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'master']], 'google-drive': [['folder_ids', 'Folder IDs', 'folder123, folder456'], ['shared_drive_ids', 'Shared drive IDs', 'drive123'], ['include_mime', 'Include MIME groups', 'docs,md,txt']], 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], + 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] }; diff --git a/src/pkg/knowledge/connector/connector.go b/src/pkg/knowledge/connector/connector.go index f66542b913..777984bb2b 100644 --- a/src/pkg/knowledge/connector/connector.go +++ b/src/pkg/knowledge/connector/connector.go @@ -216,7 +216,7 @@ func NewRegistry() *Registry { // DefaultRegistry returns a registry with the built-in connector types // (`git`, `document`, `github-wiki`, `repo-wiki`, `confluence`, `notion`, -// `sharepoint`, `google-drive` and `context7`) registered. +// `sharepoint`, `google-drive`, `context7` and `plaud`) registered. func DefaultRegistry() *Registry { r := NewRegistry() _ = r.Register(TypeGit, newGitConnector) @@ -228,6 +228,7 @@ func DefaultRegistry() *Registry { _ = r.Register(TypeSharePoint, newSharePointConnector) _ = r.Register(TypeRepoWiki, newRepoWikiConnector) _ = r.Register(TypeContext7, newContext7Connector) + _ = r.Register(TypePlaud, newPlaudConnector) return r } diff --git a/src/pkg/knowledge/connector/connector_test.go b/src/pkg/knowledge/connector/connector_test.go index 076c37a641..33e132d104 100644 --- a/src/pkg/knowledge/connector/connector_test.go +++ b/src/pkg/knowledge/connector/connector_test.go @@ -95,7 +95,7 @@ func TestRegistryRegister(t *testing.T) { if got := r.Types(); !reflect.DeepEqual(got, []string{"another", "fake"}) { t.Fatalf("Types() = %v", got) } - if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypeRepoWiki, TypeSharePoint}) { + if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint}) { t.Fatalf("DefaultRegistry().Types() = %v", got) } } diff --git a/src/pkg/knowledge/connector/mcpstdio.go b/src/pkg/knowledge/connector/mcpstdio.go new file mode 100644 index 0000000000..6cc36e2f15 --- /dev/null +++ b/src/pkg/knowledge/connector/mcpstdio.go @@ -0,0 +1,379 @@ +package connector + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" +) + +// mcpProtocolVersion is the MCP revision hive speaks as a stdio client. +const mcpProtocolVersion = "2024-11-05" + +// mcpMaxLineBytes caps one JSON-RPC line read from an MCP server. +const mcpMaxLineBytes = 32 << 20 + +// mcpClient is one connected MCP session. Call invokes a tool and returns +// the concatenated text content of the result. +type mcpClient interface { + Call(ctx context.Context, tool string, args map[string]any) (string, error) + // Notices returns side-channel text seen so far (logging notifications + // and stderr), used to surface OAuth URLs printed by the server. + Notices() string + Close() error +} + +// mcpDialer opens a fresh MCP session. +type mcpDialer func(ctx context.Context) (mcpClient, error) + +// mcpSession speaks newline-delimited JSON-RPC 2.0 over a reader/writer +// pair. Requests are serialized; notifications from the server are kept as +// notices. Close terminates the underlying process when there is one. +type mcpSession struct { + w io.Writer + lines chan []byte + done chan struct{} + nextID atomic.Int64 + mu sync.Mutex + notice strings.Builder + stderr *lockedBuffer + closer func() error + reqMu sync.Mutex +} + +type rpcMessage struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// newMCPSession wires a session over r/w and performs the MCP handshake. +func newMCPSession(ctx context.Context, r io.Reader, w io.Writer, stderr *lockedBuffer, closer func() error) (*mcpSession, error) { + s := &mcpSession{w: w, lines: make(chan []byte, 64), done: make(chan struct{}), stderr: stderr, closer: closer} + go s.readLoop(r) + init := map[string]any{ + "protocolVersion": mcpProtocolVersion, + "capabilities": map[string]any{}, + "clientInfo": map[string]any{"name": "hive-knowledge-connector", "version": "1"}, + } + if _, err := s.request(ctx, "initialize", init); err != nil { + _ = s.Close() + return nil, err + } + _ = s.notify("notifications/initialized", nil) + return s, nil +} + +func (s *mcpSession) readLoop(r io.Reader) { + defer close(s.done) + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64<<10), mcpMaxLineBytes) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + s.lines <- []byte(line) + } +} + +func (s *mcpSession) send(v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.w.Write(append(data, '\n')) + return err +} + +func (s *mcpSession) notify(method string, params any) error { + msg := map[string]any{"jsonrpc": "2.0", "method": method} + if params != nil { + msg["params"] = params + } + return s.send(msg) +} + +// request sends one request and waits for its reply, recording any +// notifications that arrive in between. +func (s *mcpSession) request(ctx context.Context, method string, params any) (json.RawMessage, error) { + s.reqMu.Lock() + defer s.reqMu.Unlock() + id := s.nextID.Add(1) + if err := s.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil { + return nil, fmt.Errorf("mcp: sending %s: %w", method, err) + } + want := fmt.Sprintf("%d", id) + for { + select { + case <-ctx.Done(): + return nil, s.withStderr(fmt.Errorf("mcp: %s: %w", method, ctx.Err())) + case <-s.done: + // Drain replies that arrived just before the server hung up. + for { + select { + case line := <-s.lines: + if res, ok, err := s.match(line, want, method); ok { + return res, err + } + continue + default: + } + return nil, s.withStderr(fmt.Errorf("mcp: %s: server closed the connection", method)) + } + case line := <-s.lines: + if res, ok, err := s.match(line, want, method); ok { + return res, err + } + } + } +} + +// match decodes one line; ok is true when it is the reply to request id want. +func (s *mcpSession) match(line []byte, want, method string) (json.RawMessage, bool, error) { + var msg rpcMessage + if err := json.Unmarshal(line, &msg); err != nil { + return nil, false, nil + } + if len(msg.ID) == 0 || msg.Method != "" { + s.recordNotification(msg) + return nil, false, nil + } + if strings.Trim(string(msg.ID), `"`) != want { + return nil, false, nil + } + if msg.Error != nil { + return nil, true, fmt.Errorf("mcp: %s: %s (code %d)", method, msg.Error.Message, msg.Error.Code) + } + return msg.Result, true, nil +} + +func (s *mcpSession) recordNotification(msg rpcMessage) { + if len(msg.Params) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.notice.Len() < 64<<10 { + s.notice.Write(msg.Params) + s.notice.WriteByte('\n') + } +} + +func (s *mcpSession) withStderr(err error) error { + if s.stderr == nil { + return err + } + if tail := strings.TrimSpace(s.stderr.Tail(2000)); tail != "" { + return fmt.Errorf("%w (stderr: %s)", err, tail) + } + return err +} + +// Notices implements mcpClient. +func (s *mcpSession) Notices() string { + s.mu.Lock() + out := s.notice.String() + s.mu.Unlock() + if s.stderr != nil { + out += s.stderr.Tail(16 << 10) + } + return out +} + +// Call implements mcpClient: tools/call with the text content joined. +func (s *mcpSession) Call(ctx context.Context, tool string, args map[string]any) (string, error) { + if args == nil { + args = map[string]any{} + } + raw, err := s.request(ctx, "tools/call", map[string]any{"name": tool, "arguments": args}) + if err != nil { + return "", err + } + var res struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(raw, &res); err != nil { + return "", fmt.Errorf("mcp: %s: decoding result: %w", tool, err) + } + var parts []string + for _, c := range res.Content { + if c.Type == "text" && c.Text != "" { + parts = append(parts, c.Text) + } + } + text := strings.Join(parts, "\n") + if res.IsError { + if text == "" { + text = "tool reported an error" + } + return text, fmt.Errorf("mcp: %s: %s", tool, firstLine(text)) + } + return text, nil +} + +// Close implements mcpClient. +func (s *mcpSession) Close() error { + if s.closer != nil { + return s.closer() + } + return nil +} + +// lockedBuffer is a goroutine-safe byte buffer used for child stderr. +type lockedBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.buf.Len() > 256<<10 { + return len(p), nil + } + return b.buf.Write(p) +} + +// Tail returns the last n bytes written. +func (b *lockedBuffer) Tail(n int) string { + b.mu.Lock() + defer b.mu.Unlock() + s := b.buf.String() + if len(s) > n { + return s[len(s)-n:] + } + return s +} + +// stdioMCPDialer starts command as a child process per session. home, when +// non-empty, overrides HOME so the server keeps its own token store in a +// persistent directory. +func stdioMCPDialer(command, home string) mcpDialer { + return func(ctx context.Context) (mcpClient, error) { + argv, err := splitCommandLine(command) + if err != nil { + return nil, err + } + if len(argv) == 0 { + return nil, errors.New("mcp: command is empty") + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = os.Environ() + if home != "" { + if err := os.MkdirAll(home, 0o700); err != nil { + return nil, fmt.Errorf("mcp: creating home %s: %w", home, err) + } + cmd.Env = append(cmd.Env, "HOME="+home) + cmd.Dir = home + } + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr := &lockedBuffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("mcp: starting %q: %w", argv[0], err) + } + closer := sync.OnceValue(func() error { + _ = stdin.Close() + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + return cmd.Wait() + }) + return newMCPSession(ctx, stdout, stdin, stderr, closer) + } +} + +// splitCommandLine splits a shell-like command line on whitespace, honouring +// single and double quotes. No expansion is performed. +func splitCommandLine(s string) ([]string, error) { + var out []string + var cur strings.Builder + inWord := false + var quote rune + for _, r := range s { + switch { + case quote != 0: + if r == quote { + quote = 0 + } else { + cur.WriteRune(r) + } + case r == '\'' || r == '"': + quote = r + inWord = true + case r == ' ' || r == '\t' || r == '\n': + if inWord { + out = append(out, cur.String()) + cur.Reset() + inWord = false + } + default: + cur.WriteRune(r) + inWord = true + } + } + if quote != 0 { + return nil, fmt.Errorf("mcp: unterminated quote in command %q", s) + } + if inWord { + out = append(out, cur.String()) + } + return out, nil +} + +var httpsURLRe = regexp.MustCompile(`https://[^\s"'<>\\)\]]+`) + +// firstHTTPSURL returns the first https:// URL found in s, or "". +func firstHTTPSURL(s string) string { + return httpsURLRe.FindString(s) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if len(s) > 300 { + s = s[:300] + "…" + } + return s +} + +// ensureDir creates dir (and parents) and returns it. +func ensureDir(dir string) (string, error) { + if dir == "" { + return "", nil + } + if err := os.MkdirAll(filepath.Clean(dir), 0o700); err != nil { + return "", err + } + return dir, nil +} diff --git a/src/pkg/knowledge/connector/mcpstdio_test.go b/src/pkg/knowledge/connector/mcpstdio_test.go new file mode 100644 index 0000000000..97a63e0df1 --- /dev/null +++ b/src/pkg/knowledge/connector/mcpstdio_test.go @@ -0,0 +1,182 @@ +package connector + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// fakeMCPServer answers JSON-RPC over pipes like a stdio MCP server. tools +// maps a tool name to a handler returning (text, isError). +type fakeMCPServer struct { + tools map[string]func(args map[string]any) (string, bool) + mu sync.Mutex + calls []string + initReq map[string]any +} + +func (f *fakeMCPServer) serve(r io.Reader, w io.Writer) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64<<10), mcpMaxLineBytes) + for sc.Scan() { + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params struct { + Name string `json:"name"` + Arguments map[string]any `json:"arguments"` + Proto string `json:"protocolVersion"` + } `json:"params"` + } + if err := json.Unmarshal(sc.Bytes(), &req); err != nil { + continue + } + if len(req.ID) == 0 { + continue // notification + } + var result any + var rpcErr map[string]any + switch req.Method { + case "initialize": + f.mu.Lock() + f.initReq = map[string]any{"protocolVersion": req.Params.Proto} + f.mu.Unlock() + // Interleave a logging notification to prove the client skips it. + fmt.Fprintf(w, `{"jsonrpc":"2.0","method":"notifications/message","params":{"level":"info","data":"booting https://plaud.example/notice"}}`+"\n") + result = map[string]any{"protocolVersion": mcpProtocolVersion, "capabilities": map[string]any{}} + case "tools/call": + f.mu.Lock() + f.calls = append(f.calls, req.Params.Name) + h := f.tools[req.Params.Name] + f.mu.Unlock() + if h == nil { + rpcErr = map[string]any{"code": -32601, "message": "unknown tool " + req.Params.Name} + break + } + text, isErr := h(req.Params.Arguments) + result = map[string]any{"content": []map[string]any{{"type": "text", "text": text}}, "isError": isErr} + default: + rpcErr = map[string]any{"code": -32601, "message": "method not found"} + } + resp := map[string]any{"jsonrpc": "2.0", "id": json.RawMessage(req.ID)} + if rpcErr != nil { + resp["error"] = rpcErr + } else { + resp["result"] = result + } + b, _ := json.Marshal(resp) + fmt.Fprintf(w, "%s\n", b) + } +} + +func (f *fakeMCPServer) callNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.calls...) +} + +// pipeDialer connects an mcpSession to srv over in-memory pipes. +func pipeDialer(srv *fakeMCPServer) mcpDialer { + return func(ctx context.Context) (mcpClient, error) { + cr, sw := io.Pipe() // server → client + sr, cw := io.Pipe() // client → server + go srv.serve(sr, sw) + closer := func() error { _ = cw.Close(); _ = sw.Close(); return nil } + return newMCPSession(ctx, cr, cw, nil, closer) + } +} + +func TestMCPSessionHandshakeCallAndErrors(t *testing.T) { + srv := &fakeMCPServer{tools: map[string]func(map[string]any) (string, bool){ + "echo": func(a map[string]any) (string, bool) { return fmt.Sprint(a["x"]), false }, + "boom": func(map[string]any) (string, bool) { return "nope: not authenticated", true }, + }} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := pipeDialer(srv)(ctx) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if got, err := c.Call(ctx, "echo", map[string]any{"x": "hi"}); err != nil || got != "hi" { + t.Fatalf("echo = %q, %v", got, err) + } + if _, err := c.Call(ctx, "boom", nil); err == nil || !strings.Contains(err.Error(), "not authenticated") { + t.Fatalf("isError must surface as error, got %v", err) + } + if _, err := c.Call(ctx, "missing", nil); err == nil || !strings.Contains(err.Error(), "unknown tool") { + t.Fatalf("rpc error = %v", err) + } + if !strings.Contains(c.Notices(), "plaud.example/notice") { + t.Fatalf("notification not recorded: %q", c.Notices()) + } + if srv.initReq["protocolVersion"] != mcpProtocolVersion { + t.Fatalf("initialize params = %v", srv.initReq) + } +} + +func TestMCPSessionServerExit(t *testing.T) { + cr, sw := io.Pipe() + sr, cw := io.Pipe() + go func() { _, _ = io.Copy(io.Discard, sr) }() + go func() { + // Reply to initialize, then hang up before any tool call is answered. + fmt.Fprintln(sw, `{"jsonrpc":"2.0","id":1,"result":{}}`) + _ = sw.Close() + }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + s, err := newMCPSession(ctx, cr, cw, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := s.Call(ctx, "x", nil); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("want closed-connection error, got %v", err) + } +} + +func TestSplitCommandLine(t *testing.T) { + got, err := splitCommandLine(`npx -y "@plaud-ai/mcp@latest" --flag 'a b'`) + if err != nil { + t.Fatal(err) + } + want := []string{"npx", "-y", "@plaud-ai/mcp@latest", "--flag", "a b"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("split = %q", got) + } + if _, err := splitCommandLine(`npx "unterminated`); err == nil { + t.Fatal("unterminated quote must fail") + } +} + +func TestStdioMCPDialerSpawnsProcess(t *testing.T) { + // A tiny shell MCP server: answers initialize and one tools/call. + dir := t.TempDir() + script := filepath.Join(dir, "server.sh") + body := "read line\necho '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}'\nread line\nread line\necho '{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"pong\"}]}}'\n" + if err := os.WriteFile(script, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + dial := stdioMCPDialer("sh "+script, filepath.Join(dir, "home")) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + c, err := dial(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer c.Close() + if got, err := c.Call(ctx, "ping", nil); err != nil || got != "pong" { + t.Fatalf("ping = %q, %v", got, err) + } + if _, err := os.Stat(filepath.Join(dir, "home")); err != nil { + t.Fatalf("HOME dir must be created: %v", err) + } +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go new file mode 100644 index 0000000000..de0c27e432 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud.go @@ -0,0 +1,421 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// TypePlaud is the connector type for Plaud voice-note recordings, read +// through the official (free) Plaud MCP server over stdio. +const TypePlaud = "plaud" + +const ( + // plaudDefaultCommand launches the official Plaud MCP server. + plaudDefaultCommand = "npx -y @plaud-ai/mcp@latest" + // plaudDefaultSinceDays bounds the first sync. + plaudDefaultSinceDays = 30 + // plaudDefaultMaxFiles caps recordings fetched per sync. + plaudDefaultMaxFiles = 200 + // plaudListPageSize is the list_files page size requested. + plaudListPageSize = 50 + // plaudRelistOverlap re-lists this far behind the cursor so late + // transcripts and edited summaries are picked up. + plaudRelistOverlap = 24 * time.Hour + // plaudLoginWindow is how long a login session stays alive to receive + // the OAuth callback. + plaudLoginWindow = 10 * time.Minute + // plaudLoginReplyWait bounds how long Connect waits for the login tool + // to answer before returning whatever URL has surfaced so far. + plaudLoginReplyWait = 8 * time.Second + // plaudHomeInherit keeps the hive process HOME for the MCP server. + plaudHomeInherit = "inherit" +) + +// plaudConnector syncs Plaud recordings: list_files → get_transcript + +// get_note → one page per recording. Scope keys: command, home, since_days, +// max_files, include_audio, query, tags. No auth.env/auth.file: Plaud's +// OAuth token lives in the MCP server's own store ($HOME/.plaud). +type plaudConnector struct { + cfg ConnectorConfig + logger *slog.Logger + dial mcpDialer + now func() time.Time + stateDir string + + mu sync.Mutex + login *plaudLogin + auth AuthState +} + +// plaudLogin is an in-flight OAuth login kept alive for the callback. +type plaudLogin struct { + client mcpClient + cancel context.CancelFunc + result string + done chan struct{} +} + +func newPlaudConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { + c := &plaudConnector{cfg: cfg, logger: deps.logger(), now: time.Now, stateDir: deps.StateDir} + home := cfg.ScopeValue("home") + switch { + case home == plaudHomeInherit: + home = "" + case home == "" && deps.StateDir != "": + home = filepath.Join(deps.StateDir, "home") + } + if _, err := ensureDir(home); err != nil { + return nil, fmt.Errorf("plaud: %w", err) + } + command := cfg.ScopeValue("command") + if command == "" { + command = plaudDefaultCommand + } + c.dial = stdioMCPDialer(command, home) + return c, nil +} + +func (c *plaudConnector) Type() string { return TypePlaud } + +// Validate checks the scope. Plaud never takes a hive-managed credential. +func (c *plaudConnector) Validate(cfg ConnectorConfig) error { + if cfg.Auth.Configured() { + return fmt.Errorf("plaud: auth.env/auth.file are not used; Plaud signs in through its own OAuth flow (Connect in the dashboard)") + } + if cmd := cfg.ScopeValue("command"); cmd != "" { + if _, err := splitCommandLine(cmd); err != nil { + return err + } + } + if h := cfg.ScopeValue("home"); h != "" && h != plaudHomeInherit && !filepath.IsAbs(h) { + return fmt.Errorf("plaud: scope.home must be an absolute path or %q", plaudHomeInherit) + } + for _, key := range []string{"since_days", "max_files"} { + if v := cfg.ScopeValue(key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return fmt.Errorf("plaud: scope.%s must be a positive integer", key) + } + } + } + if v := cfg.ScopeValue("include_audio"); v != "" { + if _, err := strconv.ParseBool(v); err != nil { + return fmt.Errorf("plaud: scope.include_audio must be true or false") + } + } + return nil +} + +func (c *plaudConnector) scopeInt(key string, def int) int { + n, err := strconv.Atoi(c.cfg.ScopeValue(key)) + if err != nil || n <= 0 { + return def + } + return n +} + +func (c *plaudConnector) includeAudio() bool { + b, _ := strconv.ParseBool(c.cfg.ScopeValue("include_audio")) + return b +} + +// plaudCursor is the newest recording time seen, RFC3339. +func parsePlaudCursor(cur Cursor) time.Time { + t, err := time.Parse(time.RFC3339, string(cur)) + if err != nil { + return time.Time{} + } + return t +} + +// Sync lists recordings created since the cursor (minus an overlap window) +// and emits one page per recording. The listing is incremental, so unlisted +// pages are never tombstoned. +func (c *plaudConnector) Sync(ctx context.Context, cur Cursor, emit func(Page) error) (Cursor, error) { + client, err := c.dial(ctx) + if err != nil { + return cur, err + } + defer func() { _ = client.Close() }() + + since := parsePlaudCursor(cur) + if since.IsZero() { + since = c.now().Add(-time.Duration(c.scopeInt("since_days", plaudDefaultSinceDays)) * 24 * time.Hour) + } else { + since = since.Add(-plaudRelistOverlap) + } + files, err := c.listFiles(ctx, client, since, c.scopeInt("max_files", plaudDefaultMaxFiles)) + if err != nil { + return cur, err + } + newest := parsePlaudCursor(cur) + var skipped []string + for _, f := range files { + if err := ctx.Err(); err != nil { + return cur, err + } + page, err := c.fetchPage(ctx, client, f) + if err != nil { + skipped = append(skipped, fmt.Sprintf("%s: %v", f.ID, err)) + continue + } + if err := emit(page); err != nil { + return cur, err + } + if f.CreatedAt.After(newest) { + newest = f.CreatedAt + } + } + c.refreshAuth(ctx, client) + next := cur + if !newest.IsZero() { + next = Cursor(newest.UTC().Format(time.RFC3339)) + } + if len(skipped) > 0 { + return next, &PartialError{Err: fmt.Errorf("plaud: %d recording(s) skipped: %s", len(skipped), strings.Join(skipped, "; "))} + } + return next, nil +} + +// listFiles pages through list_files until a page is empty, repeats, or the +// cap is hit. Recordings older than since are dropped client-side as well, +// in case the server ignores date_from. +func (c *plaudConnector) listFiles(ctx context.Context, client mcpClient, since time.Time, maxFiles int) ([]plaudFile, error) { + seen := map[string]bool{} + var out []plaudFile + for page := 1; len(out) < maxFiles; page++ { + args := map[string]any{"page": page, "page_size": plaudListPageSize} + if !since.IsZero() { + args["date_from"] = since.UTC().Format("2006-01-02") + } + if q := c.cfg.ScopeValue("query"); q != "" { + args["query"] = q + } + text, err := client.Call(ctx, "list_files", args) + if err != nil { + return nil, plaudAuthError(err) + } + files, err := decodePlaudFiles(text) + if err != nil { + return nil, err + } + added := 0 + for _, f := range files { + if f.ID == "" || seen[f.ID] { + continue + } + seen[f.ID] = true + if !since.IsZero() && !f.CreatedAt.IsZero() && f.CreatedAt.Before(since) { + continue + } + out = append(out, f) + added++ + if len(out) >= maxFiles { + break + } + } + if added == 0 || len(files) < plaudListPageSize { + break + } + } + return out, nil +} + +// fetchPage builds the knowledge page for one recording. +func (c *plaudConnector) fetchPage(ctx context.Context, client mcpClient, f plaudFile) (Page, error) { + noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": f.ID}) + transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": f.ID}) + if noteErr != nil && trErr != nil { + return Page{}, plaudAuthError(trErr) + } + if noteErr != nil { + noteText = "" + } + if trErr != nil { + transcriptText = "" + } + note := renderPlaudNote(noteText) + transcript := renderPlaudTranscript(transcriptText) + if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { + return Page{}, fmt.Errorf("no note or transcript available yet") + } + audioURL := "" + if c.includeAudio() { + if text, err := client.Call(ctx, "get_file", map[string]any{"file_id": f.ID}); err == nil { + audioURL = plaudPresignedURL(text) + } + } + title := strings.TrimSpace(f.Name) + if title == "" { + title = "Plaud recording " + f.ID + } + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + if !f.StartAt.IsZero() { + fmt.Fprintf(&md, "- **Recorded:** %s\n", f.StartAt.UTC().Format(time.RFC3339)) + } else if !f.CreatedAt.IsZero() { + fmt.Fprintf(&md, "- **Recorded:** %s\n", f.CreatedAt.UTC().Format(time.RFC3339)) + } + if f.DurationMS > 0 { + fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(f.DurationMS) * time.Millisecond).Round(time.Second)) + } + if audioURL != "" { + fmt.Fprintf(&md, "- **Audio (24h link):** %s\n", audioURL) + } + md.WriteString("\n") + if strings.TrimSpace(note) != "" { + md.WriteString("## Summary (Plaud)\n\n") + md.WriteString(strings.TrimSpace(note)) + md.WriteString("\n\n") + } + if strings.TrimSpace(transcript) != "" { + md.WriteString("## Transcript\n\n") + md.WriteString(strings.TrimSpace(transcript)) + md.WriteString("\n") + } + attrs := map[string]string{"plaud_file_id": f.ID} + if f.DurationMS > 0 { + attrs["duration_ms"] = strconv.FormatInt(f.DurationMS, 10) + } + if !f.StartAt.IsZero() { + attrs["recorded_at"] = f.StartAt.UTC().Format(time.RFC3339) + } + if tags := c.cfg.ScopeValue("tags"); tags != "" { + attrs["tags"] = tags + } + if f.Serial != "" { + attrs["device_serial"] = f.Serial + } + updated := f.CreatedAt + if f.StartAt.After(updated) { + updated = f.StartAt + } + return Page{ + ID: f.ID, + Title: title, + Markdown: md.String(), + UpdatedAt: updated, + Path: []string{"Plaud"}, + Attrs: attrs, + }, nil +} + +// plaudAuthError rewrites "not authenticated" tool errors into a hint that +// points at the dashboard Connect button. +func plaudAuthError(err error) error { + if err == nil { + return nil + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "401") || strings.Contains(msg, "not authenticated") || strings.Contains(msg, "unauthorized") || strings.Contains(msg, "login") { + return fmt.Errorf("plaud: not connected — use Connect in the Knowledge → Integrations → Plaud card (%w)", err) + } + return err +} + +// Attrs returned by the MCP server for one recording. +type plaudFile struct { + ID string + Name string + CreatedAt time.Time + StartAt time.Time + DurationMS int64 + Serial string +} + +// decodePlaudFiles accepts a JSON array of recordings, or an object wrapping +// one under files/items/data/results/recordings. +func decodePlaudFiles(text string) ([]plaudFile, error) { + text = strings.TrimSpace(text) + if text == "" { + return nil, nil + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + if i := strings.IndexAny(text, "[{"); i > 0 { + return decodePlaudFiles(text[i:]) + } + return nil, fmt.Errorf("plaud: list_files returned non-JSON output: %s", firstLine(text)) + } + var items []any + switch v := raw.(type) { + case []any: + items = v + case map[string]any: + for _, key := range []string{"files", "items", "data", "results", "recordings"} { + if arr, ok := v[key].([]any); ok { + items = arr + break + } + } + if items == nil { + if _, ok := v["id"]; ok { + items = []any{v} + } + } + } + out := make([]plaudFile, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]any) + if !ok { + continue + } + f := plaudFile{ + ID: plaudString(m, "id", "file_id"), + Name: plaudString(m, "name", "title", "filename"), + Serial: plaudString(m, "serial_number"), + CreatedAt: plaudTime(m, "created_at", "create_time"), + StartAt: plaudTime(m, "start_at", "start_time"), + } + if d, ok := m["duration"].(float64); ok { + f.DurationMS = int64(d) + } + if f.ID != "" { + out = append(out, f) + } + } + return out, nil +} + +func plaudString(m map[string]any, keys ...string) string { + for _, k := range keys { + switch v := m[k].(type) { + case string: + if s := strings.TrimSpace(v); s != "" { + return s + } + case float64: + return strconv.FormatInt(int64(v), 10) + } + } + return "" +} + +func plaudTime(m map[string]any, keys ...string) time.Time { + for _, k := range keys { + switch v := m[k].(type) { + case string: + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, strings.TrimSpace(v)); err == nil { + return t + } + } + case float64: + // Epoch seconds or milliseconds. + if v > 1e12 { + return time.UnixMilli(int64(v)).UTC() + } + if v > 0 { + return time.Unix(int64(v), 0).UTC() + } + } + } + return time.Time{} +} diff --git a/src/pkg/knowledge/connector/plaud_auth.go b/src/pkg/knowledge/connector/plaud_auth.go new file mode 100644 index 0000000000..710ea2d291 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth.go @@ -0,0 +1,374 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// AuthState describes a connector's external sign-in. Connectors that own +// their credential flow (OAuth through a vendor MCP server) report it here; +// the dashboard renders Connect/Disconnect from it. Tokens never appear. +type AuthState struct { + // Connected is true when the upstream reports a signed-in account. + Connected bool `json:"connected"` + // Account is the signed-in identity (email or display name), if known. + Account string `json:"account,omitempty"` + // Pending is true while a login awaits the user's browser authorization. + Pending bool `json:"pending"` + // AuthURL is the authorization URL the user must open, while pending. + AuthURL string `json:"auth_url,omitempty"` + // Message is the last human-readable login/logout outcome. + Message string `json:"message,omitempty"` + CheckedAt time.Time `json:"checked_at,omitempty"` +} + +// Authenticator is optionally implemented by connectors whose credential is +// obtained interactively rather than from auth.env/auth.file. +type Authenticator interface { + // AuthStatus reports the current sign-in state. It may contact the + // upstream when refresh is true; otherwise it returns the cached state. + AuthStatus(ctx context.Context, refresh bool) AuthState + // Connect begins the sign-in flow and returns the state, including the + // URL the user must open when the flow needs a browser. + Connect(ctx context.Context) (AuthState, error) + // Disconnect signs out and revokes the stored credential upstream. + Disconnect(ctx context.Context) (AuthState, error) +} + +var _ Authenticator = (*plaudConnector)(nil) + +// AuthStatus implements Authenticator. +func (c *plaudConnector) AuthStatus(ctx context.Context, refresh bool) AuthState { + c.mu.Lock() + st := c.auth + login := c.login + c.mu.Unlock() + if login != nil { + select { + case <-login.done: + st.Pending = false + if login.result != "" { + st.Message = firstLine(login.result) + } + default: + st.Pending = true + if st.AuthURL == "" { + st.AuthURL = firstHTTPSURL(login.client.Notices()) + } + } + } + if !refresh || st.Pending { + return st + } + client, err := c.dial(ctx) + if err != nil { + st.Message = err.Error() + st.CheckedAt = c.now().UTC() + c.setAuth(st) + return st + } + defer func() { _ = client.Close() }() + c.refreshAuth(ctx, client) + c.mu.Lock() + defer c.mu.Unlock() + return c.auth +} + +// refreshAuth asks get_current_user and caches the answer. +func (c *plaudConnector) refreshAuth(ctx context.Context, client mcpClient) { + st := AuthState{CheckedAt: c.now().UTC()} + text, err := client.Call(ctx, "get_current_user", nil) + switch { + case err != nil: + st.Message = firstLine(err.Error()) + default: + st.Connected = true + st.Account = plaudAccountName(text) + } + c.setAuth(st) +} + +func (c *plaudConnector) setAuth(st AuthState) { + c.mu.Lock() + c.auth = st + c.mu.Unlock() +} + +// Connect implements Authenticator: it starts a long-lived MCP session, +// calls the server's login tool and keeps the process alive for +// plaudLoginWindow so the OAuth callback (localhost:8199) can land. The +// authorization URL is taken from the tool reply or the server's log output. +func (c *plaudConnector) Connect(ctx context.Context) (AuthState, error) { + c.mu.Lock() + if l := c.login; l != nil { + select { + case <-l.done: + c.login = nil + default: + c.mu.Unlock() + return c.AuthStatus(ctx, false), nil + } + } + c.mu.Unlock() + + loginCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), plaudLoginWindow) + client, err := c.dial(loginCtx) + if err != nil { + cancel() + return c.AuthStatus(ctx, false), err + } + l := &plaudLogin{client: client, cancel: cancel, done: make(chan struct{})} + c.mu.Lock() + c.login = l + c.auth = AuthState{Pending: true, CheckedAt: c.now().UTC()} + c.mu.Unlock() + + go c.runLogin(loginCtx, l) + + timer := time.NewTimer(plaudLoginReplyWait) + defer timer.Stop() + select { + case <-l.done: + case <-timer.C: + case <-ctx.Done(): + } + return c.AuthStatus(ctx, false), nil +} + +func (c *plaudConnector) runLogin(ctx context.Context, l *plaudLogin) { + defer close(l.done) + defer l.cancel() + text, err := l.client.Call(ctx, "login", nil) + if err != nil { + text = err.Error() + } + l.result = text + st := AuthState{CheckedAt: c.now().UTC(), Message: firstLine(text)} + if url := firstHTTPSURL(text); url != "" && err == nil { + // The tool answered with the URL only: the callback still has to + // arrive, so keep the session alive until the window closes. + c.mu.Lock() + c.auth.AuthURL = url + c.auth.Pending = true + c.auth.Message = "" + c.mu.Unlock() + if !c.waitForLogin(ctx, l.client) { + st.Message = "authorization not completed before the login window closed" + st.AuthURL = "" + } else { + c.refreshAuth(ctx, l.client) + _ = l.client.Close() + c.clearLogin(l) + return + } + } else if err == nil { + c.refreshAuth(ctx, l.client) + _ = l.client.Close() + c.clearLogin(l) + return + } + _ = l.client.Close() + c.mu.Lock() + c.auth = st + c.mu.Unlock() + c.clearLogin(l) +} + +// waitForLogin polls get_current_user until it succeeds or ctx expires. +func (c *plaudConnector) waitForLogin(ctx context.Context, client mcpClient) bool { + for { + if _, err := client.Call(ctx, "get_current_user", nil); err == nil { + return true + } + if sleepCtx(ctx, 3*time.Second) != nil { + return false + } + } +} + +func (c *plaudConnector) clearLogin(l *plaudLogin) { + c.mu.Lock() + if c.login == l { + c.login = nil + } + c.auth.Pending = false + c.mu.Unlock() +} + +// Disconnect implements Authenticator via the server's logout tool. +func (c *plaudConnector) Disconnect(ctx context.Context) (AuthState, error) { + c.mu.Lock() + if l := c.login; l != nil { + l.cancel() + c.login = nil + } + c.mu.Unlock() + client, err := c.dial(ctx) + if err != nil { + return c.AuthStatus(ctx, false), err + } + defer func() { _ = client.Close() }() + text, err := client.Call(ctx, "logout", nil) + st := AuthState{CheckedAt: c.now().UTC()} + if err != nil { + st.Message = firstLine(err.Error()) + c.setAuth(st) + return st, err + } + st.Message = firstLine(text) + if st.Message == "" { + st.Message = "signed out" + } + c.setAuth(st) + return st, nil +} + +// plaudAccountName extracts an identity from get_current_user output. +func plaudAccountName(text string) string { + text = strings.TrimSpace(text) + var m map[string]any + if err := json.Unmarshal([]byte(text), &m); err == nil { + for _, key := range []string{"email", "name", "nickname", "username", "id"} { + if s := plaudString(m, key); s != "" { + return s + } + } + if u, ok := m["user"].(map[string]any); ok { + return plaudAccountName(mustJSON(u)) + } + } + return firstLine(text) +} + +func mustJSON(v any) string { + b, _ := json.Marshal(v) + return string(b) +} + +// renderPlaudNote turns get_note output into markdown. JSON replies are +// flattened: note_list entries and content/summary/markdown fields. +func renderPlaudNote(text string) string { + text = strings.TrimSpace(text) + if text == "" || !strings.HasPrefix(text, "{") && !strings.HasPrefix(text, "[") { + return text + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return text + } + var parts []string + var walk func(v any) + walk = func(v any) { + switch t := v.(type) { + case string: + if s := strings.TrimSpace(t); s != "" { + parts = append(parts, s) + } + case []any: + for _, it := range t { + walk(it) + } + case map[string]any: + for _, key := range []string{"note_list", "notes", "markdown", "content", "summary", "text", "note"} { + if inner, ok := t[key]; ok { + walk(inner) + return + } + } + if s := plaudString(t, "title"); s != "" { + parts = append(parts, "### "+s) + } + } + } + walk(raw) + // Parsed JSON with no text means the note is not ready yet. + return strings.Join(parts, "\n\n") +} + +// renderPlaudTranscript turns get_transcript output into "[mm:ss] Speaker: +// text" lines when it is JSON; plain text passes through. +func renderPlaudTranscript(text string) string { + text = strings.TrimSpace(text) + if text == "" || !strings.HasPrefix(text, "{") && !strings.HasPrefix(text, "[") { + return text + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return text + } + var segs []any + switch t := raw.(type) { + case []any: + segs = t + case map[string]any: + for _, key := range []string{"source_list", "segments", "transcript", "items", "data"} { + if arr, ok := t[key].([]any); ok { + segs = arr + break + } + if s, ok := t[key].(string); ok { + return strings.TrimSpace(s) + } + } + } + var b strings.Builder + for _, s := range segs { + m, ok := s.(map[string]any) + if !ok { + if str, ok := s.(string); ok { + b.WriteString(str + "\n") + } + continue + } + content := plaudString(m, "text", "content", "sentence") + if content == "" { + continue + } + if ts := plaudTimestamp(m); ts != "" { + fmt.Fprintf(&b, "[%s] ", ts) + } + if sp := plaudString(m, "speaker", "speaker_name", "speaker_id"); sp != "" { + b.WriteString(sp + ": ") + } + b.WriteString(content + "\n") + } + // Parsed JSON with no segments means the transcript is not ready yet. + return b.String() +} + +// plaudTimestamp formats a segment start (ms or s) as mm:ss or h:mm:ss. +func plaudTimestamp(m map[string]any) string { + for _, key := range []string{"start_time", "start", "start_ms", "begin"} { + switch v := m[key].(type) { + case float64: + // Plaud reports segment offsets in milliseconds. + d := (time.Duration(v) * time.Millisecond).Round(time.Second) + h := int(d.Hours()) + mm := int(d.Minutes()) % 60 + ss := int(d.Seconds()) % 60 + if h > 0 { + return fmt.Sprintf("%d:%02d:%02d", h, mm, ss) + } + return fmt.Sprintf("%02d:%02d", mm, ss) + case string: + if s := strings.TrimSpace(v); s != "" { + return s + } + } + } + return "" +} + +// plaudPresignedURL pulls the presigned audio URL out of get_file output. +func plaudPresignedURL(text string) string { + var m map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(text)), &m); err == nil { + if u := plaudString(m, "presigned_url", "audio_url", "url"); u != "" { + return u + } + } + return firstHTTPSURL(text) +} diff --git a/src/pkg/knowledge/connector/plaud_test.go b/src/pkg/knowledge/connector/plaud_test.go new file mode 100644 index 0000000000..eada7e56b8 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_test.go @@ -0,0 +1,229 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func plaudTestFiles() []map[string]any { + return []map[string]any{ + {"id": "rec-1", "name": "Standup ideas", "created_at": "2026-11-10T09:00:00Z", "start_at": "2026-11-10T08:58:00Z", "duration": 125000.0, "serial_number": "PN123"}, + {"id": "rec-2", "name": "Drive home", "created_at": "2026-11-11T18:00:00Z", "duration": 61000.0}, + {"id": "rec-old", "name": "Ancient", "created_at": "2020-01-01T00:00:00Z"}, + } +} + +func newPlaudTestServer() *fakeMCPServer { + srv := &fakeMCPServer{tools: map[string]func(map[string]any) (string, bool){}} + srv.tools["list_files"] = func(a map[string]any) (string, bool) { + if p, _ := a["page"].(float64); p > 1 { + return "[]", false + } + b, _ := json.Marshal(map[string]any{"files": plaudTestFiles()}) + return string(b), false + } + srv.tools["get_note"] = func(a map[string]any) (string, bool) { + if a["file_id"] == "rec-2" { + return "", true // still processing + } + b, _ := json.Marshal(map[string]any{"note_list": []string{"## Key points\n- ship plaud connector"}}) + return string(b), false + } + srv.tools["get_transcript"] = func(a map[string]any) (string, bool) { + if a["file_id"] == "rec-2" { + return `{"source_list":[]}`, false + } + return `{"source_list":[{"start_time":1500,"speaker":"Andy","text":"Hello hive"},{"start_time":65000,"speaker":"Andy","text":"Second line"}]}`, false + } + srv.tools["get_current_user"] = func(map[string]any) (string, bool) { + return `{"email":"andy@example.com"}`, false + } + return srv +} + +func newTestPlaud(t *testing.T, srv *fakeMCPServer, scope map[string]string) *plaudConnector { + t.Helper() + cfg := ConnectorConfig{Name: "plaud", Type: TypePlaud, Enabled: true, Layer: "personal", Scope: scope} + c, err := DefaultRegistry().New(cfg, Deps{StateDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + pc := c.(*plaudConnector) + pc.dial = pipeDialer(srv) + pc.now = func() time.Time { return time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) } + return pc +} + +func TestPlaudSyncEmitsPagesAndAdvancesCursor(t *testing.T) { + srv := newPlaudTestServer() + pc := newTestPlaud(t, srv, map[string]string{"tags": "voice, ideas"}) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var pages []Page + next, err := pc.Sync(ctx, "", func(p Page) error { pages = append(pages, p); return nil }) + if err == nil || !IsPartial(err) || !strings.Contains(err.Error(), "rec-2") { + t.Fatalf("want partial error naming rec-2, got %v", err) + } + if len(pages) != 1 || pages[0].ID != "rec-1" { + t.Fatalf("pages = %+v", pages) + } + p := pages[0] + for _, want := range []string{"# Standup ideas", "**Recorded:** 2026-11-10T08:58:00Z", "**Duration:** 2m5s", "## Summary (Plaud)", "ship plaud connector", "## Transcript", "[00:02] Andy: Hello hive", "[01:05] Andy: Second line"} { + if !strings.Contains(p.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, p.Markdown) + } + } + if p.Attrs["plaud_file_id"] != "rec-1" || p.Attrs["tags"] != "voice, ideas" || p.Attrs["device_serial"] != "PN123" || p.Attrs["duration_ms"] != "125000" { + t.Fatalf("attrs = %v", p.Attrs) + } + if next != Cursor("2026-11-10T09:00:00Z") { + t.Fatalf("cursor = %q", next) + } + // The sync refreshes the sign-in state for the dashboard card. + if st := pc.AuthStatus(ctx, false); !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("auth after sync = %+v", st) + } + calls := strings.Join(srv.callNames(), ",") + if strings.Contains(calls, "get_file") { + t.Fatalf("audio must not be fetched by default: %s", calls) + } +} + +func TestPlaudSyncUsesCursorWindowAndAudio(t *testing.T) { + srv := newPlaudTestServer() + var dateFrom string + list := srv.tools["list_files"] + srv.tools["list_files"] = func(a map[string]any) (string, bool) { + dateFrom, _ = a["date_from"].(string) + return list(a) + } + srv.tools["get_file"] = func(map[string]any) (string, bool) { + return `{"id":"rec-1","presigned_url":"https://cdn.plaud.example/rec-1.mp3?sig=1"}`, false + } + pc := newTestPlaud(t, srv, map[string]string{"include_audio": "true"}) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var pages []Page + next, err := pc.Sync(ctx, "2026-11-11T00:00:00Z", func(p Page) error { pages = append(pages, p); return nil }) + if err == nil || !IsPartial(err) { + t.Fatalf("err = %v", err) + } + // Cursor minus the 24h overlap → 2026-11-10; rec-1 (Nov 10) is re-listed. + if dateFrom != "2026-11-10" { + t.Fatalf("date_from = %q", dateFrom) + } + if len(pages) != 1 || !strings.Contains(pages[0].Markdown, "cdn.plaud.example/rec-1.mp3") { + t.Fatalf("pages = %+v", pages) + } + // The cursor never moves backwards. + if next != Cursor("2026-11-11T00:00:00Z") { + t.Fatalf("cursor = %q", next) + } +} + +func TestPlaudSyncNotAuthenticated(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["list_files"] = func(map[string]any) (string, bool) { return "401 Not authenticated", true } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _, err := pc.Sync(ctx, "", func(Page) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "Connect") { + t.Fatalf("want connect hint, got %v", err) + } +} + +func TestPlaudValidate(t *testing.T) { + base := ConnectorConfig{Name: "plaud", Type: TypePlaud, Layer: "personal"} + reg := DefaultRegistry() + deps := Deps{StateDir: t.TempDir()} + if _, err := reg.New(base, deps); err != nil { + t.Fatalf("default config must validate: %v", err) + } + bad := []ConnectorConfig{ + {Name: "p", Type: TypePlaud, Layer: "personal", Auth: Auth{Env: "PLAUD_TOKEN"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"since_days": "zero"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"include_audio": "maybe"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"home": "relative/dir"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"command": `npx "oops`}}, + } + for i, cfg := range bad { + if _, err := reg.New(cfg, deps); err == nil { + t.Errorf("config %d must be rejected: %+v", i, cfg.Scope) + } + } +} + +func TestPlaudConnectCompletesLogin(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["login"] = func(map[string]any) (string, bool) { + return "Open https://auth.plaud.example/authorize?state=abc to sign in", false + } + srv.tools["logout"] = func(map[string]any) (string, bool) { return "Signed out", false } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + st, err := pc.Connect(ctx) + if err != nil { + t.Fatal(err) + } + // get_current_user succeeds right away in the fake, so the login + // finishes inside the reply window and the state is already connected. + if !st.Connected || st.Account != "andy@example.com" || st.Pending { + t.Fatalf("state after connect = %+v", st) + } + st, err = pc.Disconnect(ctx) + if err != nil || st.Connected || st.Message != "Signed out" { + t.Fatalf("disconnect = %+v, %v", st, err) + } +} + +func TestPlaudConnectLoginError(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["login"] = func(map[string]any) (string, bool) { return "browser could not be opened", true } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + st, err := pc.Connect(ctx) + if err != nil { + t.Fatal(err) + } + if st.Connected || st.Pending || !strings.Contains(st.Message, "browser could not be opened") { + t.Fatalf("state = %+v", st) + } +} + +func TestPlaudRenderHelpers(t *testing.T) { + if got := renderPlaudNote("plain **markdown**"); got != "plain **markdown**" { + t.Fatalf("plain note = %q", got) + } + if got := renderPlaudNote(`{"note_list":[{"title":"Action items","content":"- do it"}]}`); got != "- do it" { + t.Fatalf("nested note = %q", got) + } + if got := renderPlaudTranscript("raw transcript"); got != "raw transcript" { + t.Fatalf("plain transcript = %q", got) + } + if got := renderPlaudTranscript(`{"transcript":"as string"}`); got != "as string" { + t.Fatalf("string transcript = %q", got) + } + if got := renderPlaudTranscript(`[{"start":3725000,"text":"late"}]`); !strings.HasPrefix(got, "[1:02:05] late") { + t.Fatalf("hour timestamp = %q", got) + } + files, err := decodePlaudFiles("Here are your files:\n" + `[{"id": 7, "name": "n", "created_at": 1762848000}]`) + if err != nil || len(files) != 1 || files[0].ID != "7" || files[0].CreatedAt.Year() != 2025 { + t.Fatalf("decode with prose prefix = %+v, %v", files, err) + } + if _, err := decodePlaudFiles("no json here"); err == nil { + t.Fatal("non-JSON listing must error") + } + if got := plaudAccountName(`{"user":{"name":"Andy"}}`); got != "Andy" { + t.Fatalf("account = %q", got) + } + if got := fmt.Sprint(plaudAuthError(fmt.Errorf("mcp: list_files: Unauthorized"))); !strings.Contains(got, "not connected") { + t.Fatalf("auth error = %q", got) + } +} diff --git a/src/pkg/knowledge/connector/syncer.go b/src/pkg/knowledge/connector/syncer.go index 160749e364..efe5efc6f7 100644 --- a/src/pkg/knowledge/connector/syncer.go +++ b/src/pkg/knowledge/connector/syncer.go @@ -159,6 +159,16 @@ func (s *Syncer) update(name string, fn func(*Status)) Status { return st } +// Connector returns the built connector for name; ok is false for an +// unknown name or one whose config failed to build. +func (s *Syncer) Connector(name string) (Connector, bool) { + e, ok := s.entries[name] + if !ok || e.buildErr != nil || e.conn == nil { + return nil, false + } + return e.conn, true +} + // SyncNow runs one sync of the named connector immediately, whether or not // it is enabled. It returns ErrSyncInProgress if a sync is already running. func (s *Syncer) SyncNow(ctx context.Context, name string) (Status, error) { From 365ce451c9eb6b640ee7dacf80f00de9da971cb6 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:35:14 +0000 Subject: [PATCH 048/223] =?UTF-8?q?=F0=9F=90=9B=20hive:=20do=20not=20gate?= =?UTF-8?q?=20the=20dashboard=20bind=20on=20the=20advisory-issue=20ensure?= =?UTF-8?q?=20(#17829)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 hive: do not gate the dashboard bind on the advisory-issue ensure Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🐛 hive: add missing time import in boot_github_deps_test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...board-bind-not-gated-on-advisory-ensure.md | 1 + src/cmd/hive/boot_github_deps_test.go | 27 +++++++++++++++++++ src/cmd/hive/main.go | 16 +++++++++-- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md diff --git a/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md b/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md new file mode 100644 index 0000000000..b7cfafe241 --- /dev/null +++ b/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md @@ -0,0 +1 @@ +- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). diff --git a/src/cmd/hive/boot_github_deps_test.go b/src/cmd/hive/boot_github_deps_test.go index 438b34332c..410e612bfa 100644 --- a/src/cmd/hive/boot_github_deps_test.go +++ b/src/cmd/hive/boot_github_deps_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" @@ -281,3 +282,29 @@ func TestBootAdvisoryWithRateLimitDoesNotClassify(t *testing.T) { t.Fatalf("log:\n%s", log.String()) } } + +func TestBootAdvisoryWithBoundsBlockedEnsure(t *testing.T) { + old := advisoryEnsureBudget + advisoryEnsureBudget = 50 * time.Millisecond + defer func() { advisoryEnsureBudget = old }() + + cfg := bootAdvisoryConfig(t) + b, _ := newDepsTestBoot(t, cfg) + b.ghClient = fakeGitHubClient(t) + f := newBootAdvisoryFake() + f.deps.ensureAdvisoryIssue = func(ctx context.Context, _ *github.Client, _ string) (int, error) { + <-ctx.Done() + return 0, ctx.Err() + } + + done := make(chan struct{}) + go func() { b.bootAdvisoryWith(f.deps); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("bootAdvisory must not block on a hung advisory-issue ensure") + } + if len(b.advisoryIssues) != 0 { + t.Fatalf("advisoryIssues = %v", b.advisoryIssues) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index bd17e6e5b7..a7fca4a85b 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1652,6 +1652,11 @@ func (b *boot) newKnowledgePrimer() *knowledge.Primer { return p } +// advisoryEnsureBudget caps the startup advisory-issue ensure (and the failure +// classification after it) so a slow or unreachable GitHub cannot hold the +// dashboard bind past the kubelet liveness budget (#17827). +var advisoryEnsureBudget = 15 * time.Second + // bootAdvisory builds the notifier, infers the ACMM level, seeds the // GitHub App banner state, opens the mutation-convergence ledger and journal, // finds or creates the pinned advisory issue and seeds the embedded default @@ -1733,7 +1738,12 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { primaryRepo = b.cfg.Project.Repos[0] } if primaryRepo != "" { - num, err := deps.ensureAdvisoryIssue(b.ctx, b.ghClient, primaryRepo) + // Bounded: every GitHub App token mint against an unreachable + // GHE burns ~8s, and this runs before the dashboard binds. An + // unresolved issue is retried per eval cycle (ensurePinnedAdvisoryIssue). + ensureCtx, cancelEnsure := context.WithTimeout(b.ctx, advisoryEnsureBudget) + num, err := deps.ensureAdvisoryIssue(ensureCtx, b.ghClient, primaryRepo) + cancelEnsure() if err != nil { b.logger.Error("failed to ensure advisory issue", "repo", primaryRepo, "error", err) // GitHub returns 403 for rate limiting too — a transient @@ -1750,7 +1760,9 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { // vanished on the first Re-check with nothing fixed. // classifyGitHubAppFailure is the same verdict Re-check // uses, and it declines to raise on AppStateUnknown. - raise, diag, state := deps.classifyAppFailure(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.logger) + classifyCtx, cancelClassify := context.WithTimeout(b.ctx, advisoryEnsureBudget) + raise, diag, state := deps.classifyAppFailure(classifyCtx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.logger) + cancelClassify() if raise { b.githubAppRequired = true b.githubAppDiag, b.githubAppState = diag, state From 17cae394f6b008b4ccc3a9b7770f88c6b2fe2280 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 02:36:59 +0000 Subject: [PATCH 049/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.227.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.227.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-hub-heartbeat-403-backoff.md | 1 - changelog.d/fixed-state-persist-rename-race.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-hub-heartbeat-403-backoff.md delete mode 100644 changelog.d/fixed-state-persist-rename-race.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 1481511f99..102ac33d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.227.1) + +### Fixed + +- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). +- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). + ## 2026-11-12 (v5.227.0) ### Added diff --git a/changelog.d/fixed-hub-heartbeat-403-backoff.md b/changelog.d/fixed-hub-heartbeat-403-backoff.md deleted file mode 100644 index cf59938b9a..0000000000 --- a/changelog.d/fixed-hub-heartbeat-403-backoff.md +++ /dev/null @@ -1 +0,0 @@ -- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). diff --git a/changelog.d/fixed-state-persist-rename-race.md b/changelog.d/fixed-state-persist-rename-race.md deleted file mode 100644 index cbe357487b..0000000000 --- a/changelog.d/fixed-state-persist-rename-race.md +++ /dev/null @@ -1 +0,0 @@ -- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). From 106ef30f0aeb76fd38f12bcf533f1f0c816189b5 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:03:20 +0000 Subject: [PATCH 050/223] =?UTF-8?q?=F0=9F=90=9B=20substrate:=20garbage-col?= =?UTF-8?q?lect=20older=20suspended=20kick=20actors=20when=20snapshot=5Fon?= =?UTF-8?q?=5Fidle=20is=20on=20(#17830)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 substrate: garbage-collect older suspended kick actors when snapshot_on_idle is on Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 substrate: scope superseded-actor GC to the atespace it was created in and log via slog Review follow-up for #17830: - lastSuspended records {Name, Atespace}; the delete uses the stored atespace and is skipped (with a warning) when the agent's atespace changed between kicks, instead of targeting a name never created there. - Replace stdlib log.Printf with an injected *slog.Logger on SubstrateOptions (slog.Default() fallback), wired from the agent manager, with agent/actor/atespace/error attrs. - Document the one-snapshot-per-agent retention rule in sandbox-isolation.md, ADR 0026 and the dashboard snapshot_on_idle help. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-substrate-snapshot-on-idle-gc.md | 1 + src/crux/agent/manager_sandbox_substrate.go | 4 +- src/crux/config/agent_config_substrate.go | 8 +- src/crux/sandbox/substrate.go | 34 +++++- src/crux/sandbox/substrate_kicks.go | 18 +++ src/crux/sandbox/substrate_test.go | 108 +++++++++++++++++- .../adr/0026-sandbox-runtime-selection.md | 2 +- src/docs/sandbox-isolation.md | 2 +- .../static-src/js/config-watchdog-security.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- 10 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 changelog.d/fixed-substrate-snapshot-on-idle-gc.md diff --git a/changelog.d/fixed-substrate-snapshot-on-idle-gc.md b/changelog.d/fixed-substrate-snapshot-on-idle-gc.md new file mode 100644 index 0000000000..97a4d8f193 --- /dev/null +++ b/changelog.d/fixed-substrate-snapshot-on-idle-gc.md @@ -0,0 +1 @@ +- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). diff --git a/src/crux/agent/manager_sandbox_substrate.go b/src/crux/agent/manager_sandbox_substrate.go index 4d535bf238..861ec4936c 100644 --- a/src/crux/agent/manager_sandbox_substrate.go +++ b/src/crux/agent/manager_sandbox_substrate.go @@ -43,7 +43,9 @@ func (m *Manager) substrateLauncherLocked(cfg config.SandboxSubstrateConfig) san if m.sandboxSubstrateLauncherFactory != nil { return m.sandboxSubstrateLauncherFactory(cfg) } - return &sandbox.SubstrateLauncher{Options: SubstrateOptionsFromConfig(cfg)} + opts := SubstrateOptionsFromConfig(cfg) + opts.Logger = m.logger + return &sandbox.SubstrateLauncher{Options: opts} } // SubstrateOptionsFromConfig converts the operator's substrate block into diff --git a/src/crux/config/agent_config_substrate.go b/src/crux/config/agent_config_substrate.go index 327eb6959b..1a847d7d8e 100644 --- a/src/crux/config/agent_config_substrate.go +++ b/src/crux/config/agent_config_substrate.go @@ -106,8 +106,12 @@ type SandboxSubstrateConfig struct { // pool. WorkerSelector map[string]string `yaml:"worker_selector,omitempty" json:"worker_selector,omitempty"` // SnapshotOnIdle suspends the actor (asynchronous snapshot) after the - // kick instead of deleting it, so Substrate keeps a resumable image. - // Default false: a kick is one-shot and snapshots cost object storage. + // kick instead of deleting it, so Substrate keeps a snapshot. The + // snapshot is retained until the agent's next idle snapshot replaces it + // (at most one per agent per hive process); it is not yet resumed on the + // next kick, and actors left from before a hive restart are not + // reclaimed. Default false: a kick is one-shot and snapshots cost + // object storage. SnapshotOnIdle bool `yaml:"snapshot_on_idle,omitempty" json:"snapshot_on_idle,omitempty"` // AllowHosts are additional egress hosts (host or host:port, leading // *. wildcard allowed) the actor may reach, on top of the GitHub hosts diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go index e79f4a87d7..965a3e8471 100644 --- a/src/crux/sandbox/substrate.go +++ b/src/crux/sandbox/substrate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log/slog" "strings" "time" @@ -105,6 +106,16 @@ type SubstrateOptions struct { SnapshotOnIdle bool AllowHosts []string PolicyProfile string + // Logger receives cleanup diagnostics that do not fail the kick; nil + // means slog.Default(). + Logger *slog.Logger +} + +func (o SubstrateOptions) logger() *slog.Logger { + if o.Logger != nil { + return o.Logger + } + return slog.Default() } // Validate reports the first configuration error a kick would hit. @@ -302,9 +313,11 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e kick := kicks.begin(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace, State: SubstrateStateCreated}) kicks.observe(kick, created) // Delete is the default: a kick is a one-shot and keeping a snapshot - // around costs object storage. snapshot_on_idle keeps a resumable - // snapshot instead, so the next kick's warm state (caches, builds) is - // a resume rather than a cold boot. + // around costs object storage. snapshot_on_idle keeps the actor + // suspended instead. Nothing resumes it yet (the next kick is a cold + // boot), so only the latest snapshot per agent is retained: the one it + // replaces is deleted. The Control API has no ListActors, so actors left + // by an earlier hive process are not reclaimed. cleanupActor := func() { if l.Options.SnapshotOnIdle && ctx.Err() == nil { if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { @@ -312,6 +325,21 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true }) cleanupTemplate() + prev := kicks.swapSuspended(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace}) + switch { + case prev.Name == "" || prev.Name == name: + case prev.Atespace != ref.Atespace: + // This launcher's client is for the current atespace; + // an actor left in another one is not ours to reach. + l.Options.logger().Warn("substrate: superseded suspended actor left in another atespace", + "agent", spec.Name, "actor", prev.Name, "atespace", prev.Atespace, "current_atespace", ref.Atespace) + default: + prevRef := &substrateapi.ObjectRef{Atespace: prev.Atespace, Name: prev.Name} + if err := control.DeleteActor(live(), prevRef); err != nil { + l.Options.logger().Warn("substrate: deleting superseded suspended actor", + "agent", spec.Name, "actor", prev.Name, "atespace", prev.Atespace, "error", err) + } + } return } } diff --git a/src/crux/sandbox/substrate_kicks.go b/src/crux/sandbox/substrate_kicks.go index 1dffb978de..cd7aba007b 100644 --- a/src/crux/sandbox/substrate_kicks.go +++ b/src/crux/sandbox/substrate_kicks.go @@ -73,6 +73,10 @@ var _ SubstrateLogReader = (*SubstrateKicks)(nil) type SubstrateKicks struct { mu sync.Mutex byLaunch map[string]*substrateKick + // lastSuspended is the newest actor each launch name left suspended + // (snapshot_on_idle), with the atespace it was created in; in memory + // only, so a hive restart forgets it. + lastSuspended map[string]SubstrateActorInfo } // substrateKick is one recorded kick: every state snapshot it went through @@ -96,6 +100,20 @@ func (s *SubstrateKicks) begin(launchName string, info SubstrateActorInfo) *subs return k } +// swapSuspended records actor (name and atespace) as launchName's retained +// suspended actor and returns the one it replaces, or a zero value when +// there was none. +func (s *SubstrateKicks) swapSuspended(launchName string, actor SubstrateActorInfo) SubstrateActorInfo { + s.mu.Lock() + defer s.mu.Unlock() + if s.lastSuspended == nil { + s.lastSuspended = map[string]SubstrateActorInfo{} + } + prev := s.lastSuspended[launchName] + s.lastSuspended[launchName] = SubstrateActorInfo{Name: actor.Name, Atespace: actor.Atespace} + return prev +} + // update applies fn to the kick's latest state and records the result as // a new snapshot when anything changed. func (s *SubstrateKicks) update(k *substrateKick, fn func(*SubstrateActorInfo)) { diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go index 1bdb458a75..29bee045e4 100644 --- a/src/crux/sandbox/substrate_test.go +++ b/src/crux/sandbox/substrate_test.go @@ -1,9 +1,11 @@ package sandbox import ( + "bytes" "context" "errors" "io" + "log/slog" "strings" "sync" "testing" @@ -46,6 +48,7 @@ type fakeSubstrate struct { pulledWorkspace string workers []*substrateapi.Worker listErr error + deleted []string closed bool cleanupCtxDead bool } @@ -118,8 +121,9 @@ func (f *fakeSubstrate) SuspendActor(ctx context.Context, _ *substrateapi.Object return f.suspendErr } -func (f *fakeSubstrate) DeleteActor(ctx context.Context, _ *substrateapi.ObjectRef) error { +func (f *fakeSubstrate) DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error { f.note("DeleteActor", ctx) + f.deleted = append(f.deleted, ref.GetName()) return f.deleteErr } @@ -452,6 +456,108 @@ func TestSubstrateRunSnapshotOnIdle(t *testing.T) { } } +func TestSubstrateSnapshotOnIdleDeletesSupersededActor(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + spec := substrateTestSpec() + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + first := substrateActorName(spec.Name, l.now()) + if len(fake.deleted) != 0 { + t.Fatalf("first suspend deleted %v, want nothing", fake.deleted) + } + + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("second Run: %v", err) + } + second := substrateActorName(spec.Name, l.now()) + if len(fake.deleted) != 1 || fake.deleted[0] != first || first == second { + t.Errorf("deleted = %v, want only the first actor %s (second %s survives)", fake.deleted, first, second) + } +} + +func TestSubstrateSnapshotOnIdleKeepsOtherAgentsActors(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + other := substrateTestSpec() + other.Name = "docs-sandbox" + if _, err := l.Run(context.Background(), other); err != nil { + t.Fatalf("Run other: %v", err) + } + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted %v: one agent's suspend must not delete another's actor", fake.deleted) + } +} + +func TestSubstrateSnapshotOnIdleSkipsSupersededActorInOtherAtespace(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + var logs bytes.Buffer + l.Options.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + spec := substrateTestSpec() + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + first := substrateActorName(spec.Name, l.now()) + + // The same agent is reconfigured onto another atespace: the launcher's + // client is for the new one, so the old actor must not be deleted + // through it (the delete would target a name never created there). + l.Options.Atespace = "hive-staging" + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("second Run: %v", err) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted = %v, want no delete across atespaces", fake.deleted) + } + if out := logs.String(); !strings.Contains(out, first) || !strings.Contains(out, "another atespace") || !strings.Contains(out, "agent="+spec.Name) { + t.Errorf("log = %q, want a warning naming %s and the agent", out, first) + } +} + +func TestSubstrateSnapshotOnIdleDeleteErrorLogsStructured(t *testing.T) { + fake := &fakeSubstrate{deleteErr: errors.New("not found")} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + var logs bytes.Buffer + l.Options.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + for _, unix := range []int64{1_700_000_000, 1_700_003_600} { + l.now = func() time.Time { return time.Unix(unix, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + } + out := logs.String() + if !strings.Contains(out, "level=WARN") || !strings.Contains(out, "agent=deps-sandbox") || !strings.Contains(out, "error=\"not found\"") { + t.Errorf("log = %q, want a WARN record with agent and error attrs", out) + } +} + +func TestSubstrateSnapshotOnIdleDeleteErrorDoesNotFailRun(t *testing.T) { + fake := &fakeSubstrate{deleteErr: errors.New("not found")} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + for _, unix := range []int64{1_700_000_000, 1_700_003_600} { + l.now = func() time.Time { return time.Unix(unix, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v (a failed GC delete must not fail the kick)", err) + } + } + if len(fake.deleted) != 1 { + t.Errorf("deleted = %v, want one attempted delete", fake.deleted) + } +} + func TestSubstrateRunContextCancelledMidKick(t *testing.T) { // The kick's context dies while the command runs: Pull is skipped, the // actor is deleted on a fresh context, and the caller gets ctx.Err(). diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index b2d0a8c838..d5f27798c2 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -254,7 +254,7 @@ opt-in and inert unless selected. | Go client `go.ate.dev/substrate` | **not imported**: it needs Go 1.27 (hive is on 1.26) and a very large module graph. A trimmed, wire-compatible proto subset is generated in-repo (`crux/sandbox/substrateapi`, field numbers pinned by tests); zero new Go modules, +578 KB on the `crux` binary | | image by tag or digest | digest-pinned `sandbox.substrate.image` only, enforced at boot (`AgentSandboxGateWarnings`) and at kick | | no `policy_profile` concept | passed to the actor as `HIVE_POLICY_PROFILE`; Substrate has no annotations | -| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; resume-on-next-kick is not implemented (#17697) | +| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; only the latest suspended actor per agent per hive process is retained (the previous one is deleted on the next suspend; actors from before a restart are not reclaimed); resume-on-next-kick is not implemented (#17697) | | `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | ### Cluster prerequisites and which hive clusters qualify diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 4b9cf5a4fc..a1e1779719 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -321,7 +321,7 @@ agents: **The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Credential brokering is unchanged: the agent inside the actor still receives `HTTPS_PROXY` pointing at the hive MITM proxy ([ADR-0002](adr/0002-mitm-proxy-network-enforcement.md)); Substrate's own zero-credential injection is complementary and not used by hive. -**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. +**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). At most one suspended actor is retained per agent per hive process: the previous one is deleted after the next successful suspend (skipped with a warning if the agent's `atespace` changed in between), and actors left by an earlier hive process are not reclaimed because the Control API has no `ListActors`. `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. **What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index 8ac65300be..f8cb3e1d36 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -543,7 +543,7 @@ { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, - { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Only the latest snapshot per agent is kept while hive runs (the previous one is deleted on the next suspend); each costs object storage.` }, { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, ]; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 1511394525..0b3097ddd5 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -66690,7 +66690,7 @@

Escalation Breaker

{ key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, - { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Only the latest snapshot per agent is kept while hive runs (the previous one is deleted on the next suspend); each costs object storage.` }, { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, ]; From a00f3195bf909d81af4fd4ec931fe3da7c445b36 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:03:33 +0000 Subject: [PATCH 051/223] =?UTF-8?q?=F0=9F=90=9B=20hub:=20converge=20startu?= =?UTF-8?q?pProbe=20and=20init=20image=20onto=20existing=20hosted=20deploy?= =?UTF-8?q?ments=20(#17831)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 hub: converge startupProbe and init image onto existing hosted deployments Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * docs: resync hub-server api-reference citations after server.go field insertion Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-hub-reconcile-startup-probe.md | 1 + src/docs/api-reference/hub-server.md | 58 ++-- src/pkg/hub/saas_sha_poller.go | 5 + src/pkg/hub/server.go | 3 + src/pkg/hub/startup_probe_reconcile.go | 258 ++++++++++++++++++ src/pkg/hub/startup_probe_reconcile_test.go | 136 +++++++++ 6 files changed, 432 insertions(+), 29 deletions(-) create mode 100644 changelog.d/fixed-hub-reconcile-startup-probe.md create mode 100644 src/pkg/hub/startup_probe_reconcile.go create mode 100644 src/pkg/hub/startup_probe_reconcile_test.go diff --git a/changelog.d/fixed-hub-reconcile-startup-probe.md b/changelog.d/fixed-hub-reconcile-startup-probe.md new file mode 100644 index 0000000000..71a7a69c72 --- /dev/null +++ b/changelog.d/fixed-hub-reconcile-startup-probe.md @@ -0,0 +1 @@ +- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index c8891dd864..9a605a3930 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,42 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1792` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1793` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1795` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1796` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1724` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1725` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1726` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1727` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1728` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1739` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1740` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1741` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1742` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1727` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1728` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1729` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1730` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1731` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1742` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1743` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1744` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1745` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1750` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1760` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1761` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1762` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1763` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1764` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1765` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1766` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1773` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1783` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1800` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1805` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1807` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1753` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1763` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1764` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1765` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1766` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1767` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1768` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1776` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1777` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1784` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1786` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1803` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1808` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1810` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/pkg/hub/saas_sha_poller.go b/src/pkg/hub/saas_sha_poller.go index a7997a6a39..be74e30875 100644 --- a/src/pkg/hub/saas_sha_poller.go +++ b/src/pkg/hub/saas_sha_poller.go @@ -720,6 +720,10 @@ func (s *HubServer) StartLatestSHAPoller(ctx context.Context) { // kubectl patch again (#17801). Throttled and rate-limited like the lane // above because each patch rolls a pod. s.reconcileHostAliasesIfDue() + // Converge the template startupProbe (#9692) and current init image repo + // onto hive Deployments provisioned before them, so a slow boot is not a + // permanent liveness kill-loop (#17827). Throttled and rate-limited. + s.reconcileStartupProbeIfDue() // Put #7457's auth-url / auth-response-headers onto the hive-contribute // Ingress of every hosted spoke provisioned before that fix, so // /api/contribute/me learns who is calling there too (#7517). The @@ -795,6 +799,7 @@ func (s *HubServer) pollLatestSHAsTick(ctx context.Context, now time.Time) { s.reconcileNodeHealthRBACIfDue() s.reconcilePerHiveEnvIfDue() s.reconcileHostAliasesIfDue() + s.reconcileStartupProbeIfDue() s.reapOrphanedPodsIfDue() s.retireExpiredGenerationsIfDue() s.sweepExpiredAccessIfDue() diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index b024fc42e5..aedd7a7e1f 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1213,6 +1213,9 @@ type HubServer struct { // lastHostAliasesReconcile throttles the per-cluster host_aliases // reconcile (cluster_host_aliases.go). Same mutex, same rationale. lastHostAliasesReconcile time.Time + // lastStartupProbeReconcile throttles the template startupProbe / init + // image reconcile (startup_probe_reconcile.go). Same mutex, same rationale. + lastStartupProbeReconcile time.Time // lastContributeIngressReconcile throttles the hive-contribute Ingress // auth-url reconcile (contribute_ingress_reconcile.go), which puts the // #7457 annotations onto spokes provisioned before that fix (#7517). diff --git a/src/pkg/hub/startup_probe_reconcile.go b/src/pkg/hub/startup_probe_reconcile.go new file mode 100644 index 0000000000..6ef0007df4 --- /dev/null +++ b/src/pkg/hub/startup_probe_reconcile.go @@ -0,0 +1,258 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +const ( + // startupProbeReconcileInterval throttles the sweep. Like the NET_ADMIN and + // host_aliases lanes the drift is STATIC: a Deployment either carries the + // template's startupProbe / current init image or it does not. + startupProbeReconcileInterval = 15 * time.Minute + + // startupProbeKubectlTimeout bounds each per-hive kubectl get/patch so one + // unreachable cluster cannot stall the whole sweep. + startupProbeKubectlTimeout = 15 * time.Second + + // startupProbeMaxPatchesPerCycle caps PATCHES per sweep: every patch + // changes the pod template and rolls that hive's pod once. + startupProbeMaxPatchesPerCycle = 3 + + // legacyHiveImageRepo is the pre-rename image repository. Its tags are no + // longer published, so an init container still pinned to it fails to pull + // on the next reschedule (#17827). + legacyHiveImageRepo = "ghcr.io/kubestellar/hive" + + // templateHiveImageRepo is the repository the provisioning template renders + // for every hive container and init container. + templateHiveImageRepo = bulkHiveImageRepo + + // hiveContainerName is the main container's name in the template. + hiveContainerName = "hive" +) + +// templateStartupProbe is the startupProbe the provisioning template renders +// onto the hive container (#9692), as the JSON value a patch writes. +type templateStartupProbe struct { + HTTPGet struct { + Path string `json:"path"` + Port int `json:"port"` + } `json:"httpGet"` + InitialDelaySeconds int `json:"initialDelaySeconds"` + PeriodSeconds int `json:"periodSeconds"` + FailureThreshold int `json:"failureThreshold"` +} + +func wantHostedStartupProbe() templateStartupProbe { + var p templateStartupProbe + p.HTTPGet.Path = "/api/health" + p.HTTPGet.Port = dashboardPort + p.InitialDelaySeconds = hostedStartupProbeInitialDelaySeconds + p.PeriodSeconds = hostedStartupProbePeriodSeconds + p.FailureThreshold = hostedStartupProbeFailureThreshold + return p +} + +// liveProbe is the subset of a live probe the template controls. Fields the +// API server defaults (scheme, timeoutSeconds, successThreshold) are ignored +// so a converged Deployment never compares unequal. +type liveProbe struct { + HTTPGet *struct { + Path string `json:"path"` + Port json.RawMessage `json:"port"` + } `json:"httpGet"` + InitialDelaySeconds int `json:"initialDelaySeconds"` + PeriodSeconds int `json:"periodSeconds"` + FailureThreshold int `json:"failureThreshold"` +} + +type liveContainer struct { + Name string `json:"name"` + Image string `json:"image"` + StartupProbe *liveProbe `json:"startupProbe"` +} + +type liveHiveDeployment struct { + Spec struct { + Template struct { + Spec struct { + InitContainers []liveContainer `json:"initContainers"` + Containers []liveContainer `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` +} + +func startupProbeMatches(live *liveProbe, want templateStartupProbe) bool { + if live == nil || live.HTTPGet == nil { + return false + } + port := strings.Trim(string(live.HTTPGet.Port), `"`) + return live.HTTPGet.Path == want.HTTPGet.Path && + port == strconv.Itoa(want.HTTPGet.Port) && + live.InitialDelaySeconds == want.InitialDelaySeconds && + live.PeriodSeconds == want.PeriodSeconds && + live.FailureThreshold == want.FailureThreshold +} + +// rewriteLegacyHiveImage returns the image with a legacyHiveImageRepo +// repository swapped for templateHiveImageRepo, keeping the tag (or digest). +// ok is false when the image is not on the legacy repository. +func rewriteLegacyHiveImage(image string) (string, bool) { + image = strings.TrimSpace(image) + if !strings.HasPrefix(image, legacyHiveImageRepo) { + return "", false + } + rest := image[len(legacyHiveImageRepo):] + if rest != "" && rest[0] != ':' && rest[0] != '@' { + return "", false + } + return templateHiveImageRepo + rest, true +} + +// startupProbePatchJSON is the PURE decision function for the sweep. Given the +// live Deployment JSON (`kubectl get deployment hive -o json`), it returns the +// JSON-Patch that sets the hive container's startupProbe to the template's and +// rewrites any init-container image on the legacy repository, plus a short +// description of each change. An empty patch means already converged. +func startupProbePatchJSON(deployJSON []byte) (string, []string, error) { + var d liveHiveDeployment + if err := json.Unmarshal(deployJSON, &d); err != nil { + return "", nil, fmt.Errorf("parsing deployment: %w", err) + } + podSpec := d.Spec.Template.Spec + var ops []map[string]any + var changes []string + + idx := -1 + for i, c := range podSpec.Containers { + if c.Name == hiveContainerName { + idx = i + break + } + } + if idx >= 0 { + want := wantHostedStartupProbe() + if !startupProbeMatches(podSpec.Containers[idx].StartupProbe, want) { + ops = append(ops, map[string]any{ + "op": "add", + "path": fmt.Sprintf("/spec/template/spec/containers/%d/startupProbe", idx), + "value": want, + }) + changes = append(changes, "startupProbe") + } + } + + for i, c := range podSpec.InitContainers { + img, ok := rewriteLegacyHiveImage(c.Image) + if !ok { + continue + } + ops = append(ops, map[string]any{ + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/initContainers/%d/image", i), + "value": img, + }) + changes = append(changes, "initContainers["+c.Name+"].image="+img) + } + + if len(ops) == 0 { + return "", nil, nil + } + b, err := json.Marshal(ops) + if err != nil { + return "", nil, err + } + return string(b), changes, nil +} + +// reconcileStartupProbeIfDue runs the sweep at most once per +// startupProbeReconcileInterval. Safe to call from the poller loop every tick; +// same guarding mutex as reconcileHostAliasesIfDue (poller-loop-only state). +func (s *HubServer) reconcileStartupProbeIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastStartupProbeReconcile.IsZero() || + time.Since(s.lastStartupProbeReconcile) >= startupProbeReconcileInterval + if due { + s.lastStartupProbeReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileStartupProbe() +} + +// reconcileStartupProbe converges the template's startupProbe (#9692) and the +// current init-container image repository onto hosted hive Deployments +// provisioned before those landed (#17827). Without the startupProbe a slow +// boot is killed by the livenessProbe at ~90s and never recovers. At most +// startupProbeMaxPatchesPerCycle patches per cycle, since each rolls a pod. +func (s *HubServer) reconcileStartupProbe() { + patched := 0 + deferred := 0 + for _, h := range listSaaSHives() { + if !netAdminSweepEligible(h.Status) { + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil || s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + ns := hiveHostedNamespacePrefix + h.ID + + ctx, cancel := context.WithTimeout(context.Background(), startupProbeKubectlTimeout) + getCmd := kubectlForClusterContext(ctx, cluster, "get", "deployment", "hive", + "-n", ns, "-o", "json") + out, err := getCmd.Output() + cancel() + if err != nil { + s.logger.Debug("startupProbe reconcile: could not read hive deployment", + "hive_id", h.ID, "cluster", cluster.ID, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + patch, changes, perr := startupProbePatchJSON(out) + if perr != nil { + s.logger.Warn("startupProbe reconcile: could not build patch", + "hive_id", h.ID, "cluster", cluster.ID, "error", perr) + continue + } + if patch == "" { + continue + } + if patched >= startupProbeMaxPatchesPerCycle { + deferred++ + continue + } + pctx, pcancel := context.WithTimeout(context.Background(), startupProbeKubectlTimeout) + patchCmd := kubectlForClusterContext(pctx, cluster, "patch", "deployment", "hive", + "-n", ns, "--type=json", "-p", patch) + pout, err := patchCmd.CombinedOutput() + pcancel() + if err != nil { + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("startupProbe reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, + "output", strings.TrimSpace(string(pout)), "error", err) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled template startupProbe/init image onto hive deployment (rolls the pod once)", + "hive_id", h.ID, "cluster", cluster.ID, + "changes", strings.Join(changes, ","), "patched_this_cycle", patched) + } + if deferred > 0 { + s.logger.Info("startupProbe reconcile: rate limit reached, remaining hives deferred to next cycle", + "patched", patched, "deferred", deferred, + "max_per_cycle", startupProbeMaxPatchesPerCycle, + "next_cycle_in", startupProbeReconcileInterval.String()) + } +} diff --git a/src/pkg/hub/startup_probe_reconcile_test.go b/src/pkg/hub/startup_probe_reconcile_test.go new file mode 100644 index 0000000000..a250e9af05 --- /dev/null +++ b/src/pkg/hub/startup_probe_reconcile_test.go @@ -0,0 +1,136 @@ +package hub + +import ( + "encoding/json" + "strings" + "testing" +) + +type testPatchOp struct { + Op string `json:"op"` + Path string `json:"path"` + Value json.RawMessage `json:"value"` +} + +func decodeStartupProbePatch(t *testing.T, patch string) []testPatchOp { + t.Helper() + var ops []testPatchOp + if err := json.Unmarshal([]byte(patch), &ops); err != nil { + t.Fatalf("patch is not valid JSON-Patch: %v\n%s", err, patch) + } + return ops +} + +func TestStartupProbePatchAddsMissingProbe(t *testing.T) { + live := []byte(`{"spec":{"template":{"spec":{ + "initContainers":[{"name":"copy-config","image":"ghcr.io/hivecommons/hive:v4-latest"}], + "containers":[ + {"name":"sidecar","image":"busybox"}, + {"name":"hive","image":"ghcr.io/hivecommons/hive:v4-latest", + "livenessProbe":{"httpGet":{"path":"/api/livez","port":3002},"periodSeconds":30,"failureThreshold":3}} + ]}}}}`) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + ops := decodeStartupProbePatch(t, patch) + if len(ops) != 1 || ops[0].Op != "add" || ops[0].Path != "/spec/template/spec/containers/1/startupProbe" { + t.Fatalf("unexpected patch: %s", patch) + } + var got templateStartupProbe + if err := json.Unmarshal(ops[0].Value, &got); err != nil { + t.Fatal(err) + } + if got != wantHostedStartupProbe() { + t.Fatalf("patched probe %+v != template %+v", got, wantHostedStartupProbe()) + } + if got.HTTPGet.Path != "/api/health" || got.HTTPGet.Port != dashboardPort || got.FailureThreshold != hostedStartupProbeFailureThreshold { + t.Fatalf("probe does not match the template: %+v", got) + } + if len(changes) != 1 || changes[0] != "startupProbe" { + t.Fatalf("unexpected changes: %v", changes) + } +} + +func TestStartupProbePatchReplacesDriftedProbe(t *testing.T) { + live := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive", + "startupProbe":{"httpGet":{"path":"/api/health","port":3002},"initialDelaySeconds":10,"periodSeconds":5,"failureThreshold":30}}]}}}}`) + patch, _, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + if ops := decodeStartupProbePatch(t, patch); len(ops) != 1 || ops[0].Path != "/spec/template/spec/containers/0/startupProbe" { + t.Fatalf("drifted probe must be replaced: %s", patch) + } +} + +func TestStartupProbePatchNoopWhenConverged(t *testing.T) { + want := wantHostedStartupProbe() + // API-server defaulted fields (scheme, timeoutSeconds, successThreshold) + // must not make a converged Deployment look drifted. + probe := map[string]any{ + "httpGet": map[string]any{"path": want.HTTPGet.Path, "port": want.HTTPGet.Port, "scheme": "HTTP"}, + "initialDelaySeconds": want.InitialDelaySeconds, + "periodSeconds": want.PeriodSeconds, + "failureThreshold": want.FailureThreshold, + "timeoutSeconds": 1, + "successThreshold": 1, + } + dep := map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{ + "initContainers": []any{map[string]any{"name": "init-permissions", "image": "ghcr.io/hivecommons/hive:v4-latest"}}, + "containers": []any{map[string]any{"name": "hive", "startupProbe": probe}}, + }}}} + live, _ := json.Marshal(dep) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + if patch != "" || len(changes) != 0 { + t.Fatalf("converged deployment must not be patched, got %s (%v)", patch, changes) + } +} + +func TestStartupProbePatchRewritesLegacyInitImageKeepingTag(t *testing.T) { + want, _ := json.Marshal(wantHostedStartupProbe()) + live := []byte(`{"spec":{"template":{"spec":{ + "initContainers":[ + {"name":"copy-config","image":"ghcr.io/hivecommons/hive:v4-latest"}, + {"name":"init-permissions","image":"ghcr.io/kubestellar/hive:v2-latest"} + ], + "containers":[{"name":"hive","startupProbe":` + string(want) + `}]}}}}`) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + ops := decodeStartupProbePatch(t, patch) + if len(ops) != 1 || ops[0].Op != "replace" || ops[0].Path != "/spec/template/spec/initContainers/1/image" { + t.Fatalf("unexpected patch: %s", patch) + } + var img string + if err := json.Unmarshal(ops[0].Value, &img); err != nil { + t.Fatal(err) + } + if img != "ghcr.io/hivecommons/hive:v2-latest" { + t.Fatalf("legacy init image must move to the template repo keeping its tag, got %q", img) + } + if len(changes) != 1 || !strings.Contains(changes[0], "init-permissions") { + t.Fatalf("unexpected changes: %v", changes) + } +} + +func TestRewriteLegacyHiveImage(t *testing.T) { + for in, want := range map[string]string{ + "ghcr.io/kubestellar/hive:v2-latest": "ghcr.io/hivecommons/hive:v2-latest", + "ghcr.io/kubestellar/hive": "ghcr.io/hivecommons/hive", + "ghcr.io/kubestellar/hive@sha256:abc": "ghcr.io/hivecommons/hive@sha256:abc", + } { + if got, ok := rewriteLegacyHiveImage(in); !ok || got != want { + t.Errorf("%s: got %q,%v want %q", in, got, ok, want) + } + } + for _, in := range []string{"ghcr.io/hivecommons/hive:v4-latest", "ghcr.io/kubestellar/hive-other:v1", "busybox"} { + if _, ok := rewriteLegacyHiveImage(in); ok { + t.Errorf("%s must not be rewritten", in) + } + } +} From 898150cf18367977275e8e98d5687b10ce00ecf1 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 03:18:56 +0000 Subject: [PATCH 052/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-knowledge-plaud-source.md | 1 - ...ixed-dashboard-bind-not-gated-on-advisory-ensure.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-knowledge-plaud-source.md delete mode 100644 changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 102ac33d96..05b761601f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.0) + +### Added + +- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. + +### Fixed + +- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). + ## 2026-11-12 (v5.227.1) ### Fixed diff --git a/changelog.d/added-knowledge-plaud-source.md b/changelog.d/added-knowledge-plaud-source.md deleted file mode 100644 index c7cf5b02d6..0000000000 --- a/changelog.d/added-knowledge-plaud-source.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. diff --git a/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md b/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md deleted file mode 100644 index b7cfafe241..0000000000 --- a/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md +++ /dev/null @@ -1 +0,0 @@ -- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). From 61fae9ce52fff83ade2e0052c9ba607abe78997e Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 20:24:26 -0700 Subject: [PATCH 053/223] =?UTF-8?q?=F0=9F=90=9B=20hub:=20negatively=20cach?= =?UTF-8?q?e=20rejected=20bearer=20tokens;=20stop=20re-probing=20exhausted?= =?UTF-8?q?=20user=20quota=20(#17837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateGitHubToken re-hit GET /user with the caller's own token on every request once the token was rejected or rate-limited, charging the owner's exhausted core bucket and prolonging the 401 outage. Remember 401/403/429 for 60 s (or until X-RateLimit-Reset, capped at 15 min), record the probe in REST accounting under the pass-through caller agent:hub-bearer-validate, and route hub release-notes contents reads through the shared ETag/accounting transport. Document the fleet-wide operator steps for getting personal tokens out of cluster secrets (ARC runner scale set -> GitHub App auth). Signed-off-by: Andrew Anderson --- .../fixed-hub-bearer-negative-cache.md | 1 + .../operator-reference/runtime-and-quota.md | 51 ++++++ src/pkg/hub/release_notes.go | 5 +- src/pkg/hub/saas_auth.go | 70 +++++++- src/pkg/hub/saas_auth_bearer_cache_test.go | 159 ++++++++++++++++++ 5 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-hub-bearer-negative-cache.md create mode 100644 src/pkg/hub/saas_auth_bearer_cache_test.go diff --git a/changelog.d/fixed-hub-bearer-negative-cache.md b/changelog.d/fixed-hub-bearer-negative-cache.md new file mode 100644 index 0000000000..2fde88abb7 --- /dev/null +++ b/changelog.d/fixed-hub-bearer-negative-cache.md @@ -0,0 +1 @@ +- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index dd3d67c23f..f85565eff8 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -183,6 +183,57 @@ loops, so the proxy enforces two safeguards: once. Bodies are stored gzip-compressed, so the cap is several times its size in raw JSON. +### Hub bearer validation and exhausted user tokens + +The hub validates a `Bearer` token it has not seen before with one +`GET /user` **using that token**, then caches the login for 5 minutes. A token +GitHub rejects (401, or 403/429 with `X-RateLimit-Remaining: 0`) is now also +remembered: for 60 s by default, or until the token's `X-RateLimit-Reset` +(capped at 15 minutes) when it is rate-limited. Without that negative entry a +client retrying with an exhausted token made the hub re-probe GitHub on every +request, charging the token owner's own core bucket while it was already at +zero. The probe is recorded in `/api/gh-rate-limits` under caller +`agent:hub-bearer-validate`, so a flood of 401s from one dead token is visible +on the hub admin page; being a pass-through credential it never updates the +hub's own core-bucket snapshot. + +### Keep personal tokens out of cluster secrets + +GitHub's 5,000 req/h core limit is **per user**, shared by every PAT and +OAuth token (`gho_…`, `ghp_…`, `github_pat_…`) that user owns, from every +host. A `gh auth token` pasted into a cluster secret therefore shares its +owner's bucket with their laptop `gh`, the hub's bearer validation of that +user, and any other service holding a copy. The hub itself authenticates as +the GitHub App (`HIVE_HUB_GITHUB_APP_ID` / `HIVE_HUB_GITHUB_INSTALLATION_ID` / +`HIVE_HUB_GITHUB_APP_KEY_FILE`, see [env-vars.md](../env-vars.md)) and uses +`HIVE_HUB_GITHUB_TOKEN` only as a fallback, logging a startup `WARN` naming the +identity it ended up with. Audit the rest of the fleet the same way: + +1. **Find copies.** For every cluster context, compare the first 8 characters + of each GitHub-looking secret value and `/data/gh-user-token` file against + the owner's token (`kubectl get secret -A -o json`, decode, never print the + full value). Known holders in a hive deployment are the Actions Runner + Controller (`gha-runner-scale-set`) `githubConfigSecret`, its generated + `*-listener-config` secret, and any sidecar that scrapes GitHub (dibs). +2. **Move ARC to App auth.** The runner scale-set listener long-polls the + Actions broker and acknowledges every message; with a PAT those calls are + charged to the PAT owner (~3,000 req/h at 350 ephemeral runners/h). Grant + the GitHub App repository **Administration: read & write** (or organization + **Self-hosted runners: read & write** for an org-level runner group), + then replace the `githubConfigSecret` keys `github_token` with + `github_app_id`, `github_app_installation_id` and + `github_app_private_key` (the same PEM mounted on the hub as + `hive-hub-github-app-key`), and delete the listener pod so the controller + regenerates its `*-listener-config` from the new secret. App traffic lands + in the installation's own 15,000 req/h bucket. +3. **Remove the PAT from the hub.** With the three App variables set, unset + `HIVE_HUB_GITHUB_TOKEN` on `deploy/hive-hub`; the startup log then reads + `hub GitHub API identity selected mode=app` instead of the `WARN`, and + `GET /api/gh-rate-limits` reports the App installation's `core.limit`. +4. **Rotate.** Once no cluster holds it, revoke the leaked user token + (`gh auth refresh` mints a new `gho_`), so stale copies fail closed rather + than silently sharing the bucket again. + The proxy returns JSON shaped for the GitHub CLI: ```json diff --git a/src/pkg/hub/release_notes.go b/src/pkg/hub/release_notes.go index 82bc94d3e2..8e621cd090 100644 --- a/src/pkg/hub/release_notes.go +++ b/src/pkg/hub/release_notes.go @@ -13,6 +13,7 @@ import ( "sync" "time" + hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/releasenotes" ) @@ -30,7 +31,9 @@ const ( var ( hubReleaseNotesRawBaseURL = "https://raw.githubusercontent.com" - hubReleaseNotesHTTP = &http.Client{Timeout: 10 * time.Second} + // Contents reads go through pkg/github's transport so repeat fetches of an + // unchanged ref are ETag 304s and the calls show up in /api/gh-rate-limits. + hubReleaseNotesHTTP = &http.Client{Transport: hgithub.NewHTTPTransport(nil), Timeout: 10 * time.Second} hubReleaseNotesSHAPattern = regexp.MustCompile(`^[0-9a-fA-F]{7,40}$`) ) diff --git a/src/pkg/hub/saas_auth.go b/src/pkg/hub/saas_auth.go index d493b6fee6..5c0a7d5281 100644 --- a/src/pkg/hub/saas_auth.go +++ b/src/pkg/hub/saas_auth.go @@ -8,11 +8,14 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "sync" "time" "golang.org/x/net/publicsuffix" + + hgithub "github.com/hivecommons/hive/pkg/github" ) func (s *HubServer) requireAuth(next http.HandlerFunc) http.HandlerFunc { @@ -691,13 +694,60 @@ var ( ghTokenCache = map[string]ghTokenCacheEntry{} ) -const ghTokenCacheTTL = 5 * time.Minute +const ( + ghTokenCacheTTL = 5 * time.Minute + // ghTokenNegativeTTL bounds how long a rejected bearer token (401/403, + // including a rate-limited one) is remembered without re-probing + // GET /user. Without it every request carrying a dead or exhausted token + // re-hit GitHub, charging the token owner's own core quota while it was + // already at zero and stretching the outage. + ghTokenNegativeTTL = 60 * time.Second + // ghTokenNegativeMaxTTL caps how far a 403's X-RateLimit-Reset may push + // the negative entry so a skewed clock cannot lock a user out for long. + ghTokenNegativeMaxTTL = 15 * time.Minute + ghTokenValidateTimeout = 10 * time.Second + ghTokenValidateCaller = "agent:hub-bearer-validate" +) type ghTokenCacheEntry struct { username string expiresAt time.Time } +// ghTokenValidateHTTPClient probes GET /user for bearer validation. It is a +// plain client on purpose: the probe carries the CALLER's token, and routing +// it through pkg/github's shared transport chain would let one exhausted user +// bucket push the hub's own App traffic into rate-limit caution. The probe is +// still recorded in REST accounting (see validateGitHubToken). +func ghTokenValidateHTTPClient() *http.Client { + return &http.Client{Timeout: ghTokenValidateTimeout} +} + +// ghTokenNegativeExpiry picks the expiry of a negative cache entry for a +// rejected probe: the default short TTL, extended up to ghTokenNegativeMaxTTL +// when GitHub says the token is rate-limited until X-RateLimit-Reset. +func ghTokenNegativeExpiry(status int, hdr http.Header, now time.Time) time.Time { + exp := now.Add(ghTokenNegativeTTL) + if status != http.StatusForbidden && status != http.StatusTooManyRequests { + return exp + } + if hdr.Get("X-RateLimit-Remaining") != "0" { + return exp + } + reset, err := strconv.ParseInt(strings.TrimSpace(hdr.Get("X-RateLimit-Reset")), 10, 64) + if err != nil { + return exp + } + until := time.Unix(reset, 0) + if until.After(now.Add(ghTokenNegativeMaxTTL)) { + until = now.Add(ghTokenNegativeMaxTTL) + } + if until.After(exp) { + exp = until + } + return exp +} + func (s *HubServer) validateGitHubToken(token string) string { if token == "" { return "" @@ -710,19 +760,31 @@ func (s *HubServer) validateGitHubToken(token string) string { } ghTokenCacheMu.RUnlock() - client := &http.Client{Timeout: 10 * time.Second} + client := ghTokenValidateHTTPClient() // Hub always validates tokens against github.com (the hub is a SaaS service). - req, err := http.NewRequest("GET", defaultGHUserURL, nil) + req, err := http.NewRequest(http.MethodGet, defaultGHUserURL, nil) if err != nil { return "" } req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Accept", "application/vnd.github+json") resp, err := client.Do(req) - if err != nil || resp.StatusCode != 200 { + if err != nil { return "" } defer func() { _ = resp.Body.Close() }() + // Attributed under the pass-through ("agent:") caller prefix: visible in + // /api/gh-rate-limits, but the caller's rate-limit headers never overwrite + // the hub's own core-bucket snapshot. + hgithub.RecordRESTRequest(ghTokenValidateCaller, req.Method, req.URL.Path, resp.StatusCode, resp.Header) + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { + ghTokenCacheMu.Lock() + ghTokenCache[token] = ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())} + ghTokenCacheMu.Unlock() + } + return "" + } var user struct { Login string `json:"login"` } diff --git a/src/pkg/hub/saas_auth_bearer_cache_test.go b/src/pkg/hub/saas_auth_bearer_cache_test.go new file mode 100644 index 0000000000..b9c62866e5 --- /dev/null +++ b/src/pkg/hub/saas_auth_bearer_cache_test.go @@ -0,0 +1,159 @@ +package hub + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" + + hgithub "github.com/hivecommons/hive/pkg/github" +) + +func withBearerProbeServer(t *testing.T, handler http.HandlerFunc) *int32 { + t.Helper() + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + handler(w, r) + })) + oldURL := defaultGHUserURL + defaultGHUserURL = srv.URL + "/user" + t.Cleanup(func() { + defaultGHUserURL = oldURL + srv.Close() + }) + return &hits +} + +func clearGHTokenCache(t *testing.T, tokens ...string) { + t.Helper() + clear := func() { + ghTokenCacheMu.Lock() + for _, tok := range tokens { + delete(ghTokenCache, tok) + } + ghTokenCacheMu.Unlock() + } + clear() + t.Cleanup(clear) +} + +// A rejected bearer token must not re-probe GET /user on every request while +// the negative entry is fresh: that re-probe charged the token owner's own +// core quota while it was already exhausted (the 401 storm in the PAT outage). +func TestValidateGitHubToken_RejectedTokenIsNegativelyCached(t *testing.T) { + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }) + const token = "dead-token-negative-cache" + clearGHTokenCache(t, token) + + srv := newHubServerForTest(t) + for i := 0; i < 3; i++ { + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("call %d: rejected token resolved to %q", i, got) + } + } + if n := atomic.LoadInt32(hits); n != 1 { + t.Fatalf("GET /user probed %d times for a rejected token; want 1", n) + } + ghTokenCacheMu.RLock() + entry, ok := ghTokenCache[token] + ghTokenCacheMu.RUnlock() + if !ok || entry.username != "" || !entry.expiresAt.After(time.Now()) { + t.Fatalf("negative entry = %+v, ok=%v; want an unexpired entry with empty username", entry, ok) + } +} + +// An expired negative entry probes again, and a token that has since become +// valid resolves — the negative cache is a short hold, not a ban. +func TestValidateGitHubToken_ExpiredNegativeEntryReprobes(t *testing.T) { + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"login": "revived"}) + }) + const token = "revived-token" + clearGHTokenCache(t, token) + ghTokenCacheMu.Lock() + ghTokenCache[token] = ghTokenCacheEntry{expiresAt: time.Now().Add(-time.Second)} + ghTokenCacheMu.Unlock() + + srv := newHubServerForTest(t) + if got := srv.validateGitHubToken(token); got != "revived" { + t.Fatalf("validateGitHubToken = %q, want revived", got) + } + if n := atomic.LoadInt32(hits); n != 1 { + t.Fatalf("GET /user probed %d times; want 1", n) + } +} + +func TestGhTokenNegativeExpiry(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + hdr := func(remaining string, reset time.Time) http.Header { + h := http.Header{} + h.Set("X-RateLimit-Remaining", remaining) + h.Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + return h + } + cases := []struct { + name string + status int + hdr http.Header + want time.Time + }{ + {"401 uses the short hold", http.StatusUnauthorized, http.Header{}, now.Add(ghTokenNegativeTTL)}, + {"403 with quota left is not a rate limit", http.StatusForbidden, hdr("12", now.Add(time.Hour)), now.Add(ghTokenNegativeTTL)}, + {"403 exhausted holds until reset", http.StatusForbidden, hdr("0", now.Add(5*time.Minute)), now.Add(5 * time.Minute)}, + {"429 exhausted holds until reset", http.StatusTooManyRequests, hdr("0", now.Add(3*time.Minute)), now.Add(3 * time.Minute)}, + {"reset far away is capped", http.StatusForbidden, hdr("0", now.Add(2*time.Hour)), now.Add(ghTokenNegativeMaxTTL)}, + {"reset already passed keeps the short hold", http.StatusForbidden, hdr("0", now.Add(-time.Minute)), now.Add(ghTokenNegativeTTL)}, + {"garbage reset keeps the short hold", http.StatusForbidden, func() http.Header { + h := http.Header{} + h.Set("X-RateLimit-Remaining", "0") + h.Set("X-RateLimit-Reset", "soon") + return h + }(), now.Add(ghTokenNegativeTTL)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ghTokenNegativeExpiry(tc.status, tc.hdr, now); !got.Equal(tc.want) { + t.Fatalf("ghTokenNegativeExpiry = %v, want %v", got, tc.want) + } + }) + } +} + +// The probe is attributed in /api/gh-rate-limits under the pass-through +// ("agent:") caller so an exhausted user token is visible on the hub admin +// page without being mistaken for the hub's own App credential. +func TestValidateGitHubToken_ProbeIsAccountedAsPassThroughCaller(t *testing.T) { + withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10)) + w.WriteHeader(http.StatusForbidden) + }) + const token = "accounted-probe-token" + clearGHTokenCache(t, token) + + srv := newHubServerForTest(t) + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("validateGitHubToken = %q, want empty", got) + } + var found *hgithub.RESTConsumer + for _, c := range hgithub.RESTTopConsumers(100) { + if c.Caller == ghTokenValidateCaller && c.Endpoint == "/user" { + c := c + found = &c + break + } + } + if found == nil { + t.Fatalf("caller %q not found in REST accounting: %+v", ghTokenValidateCaller, hgithub.RESTTopConsumers(100)) + } + if found.RateLimited == 0 { + t.Fatalf("probe not recorded as rate-limited: %+v", *found) + } +} From 17722e1614811b0ff66baf291c17b6f6a5070caf Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:50:12 +0000 Subject: [PATCH 054/223] =?UTF-8?q?=F0=9F=90=9B=20crux=20serve:=20count=20?= =?UTF-8?q?auth-rejected=20requests=20in=20crux.serve.auth.rejections=20(#?= =?UTF-8?q?17839)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit instrument sits inside authenticate, so requests refused with a 401 never reached crux.serve.requests or crux.serve.request.duration. Wrapping authenticate with instrument would label them with the unmatched route, so a separate counter, crux.serve.auth.rejections, is incremented in each 401 branch of authenticate. Its only attribute is a fixed-enum reason (missing_or_invalid_token, project_forbidden, no_token_configured); never a token, path or remote address. Closes hivecommons/crux#146 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-crux-serve-auth-rejections.md | 1 + src/crux/serve/metrics.go | 24 ++++++++++- src/crux/serve/metrics_test.go | 41 +++++++++++++++++++ src/crux/serve/serve.go | 3 ++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-crux-serve-auth-rejections.md diff --git a/changelog.d/fixed-crux-serve-auth-rejections.md b/changelog.d/fixed-crux-serve-auth-rejections.md new file mode 100644 index 0000000000..ba4e30efba --- /dev/null +++ b/changelog.d/fixed-crux-serve-auth-rejections.md @@ -0,0 +1 @@ +- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. diff --git a/src/crux/serve/metrics.go b/src/crux/serve/metrics.go index 4ea8b016c7..e21088a79c 100644 --- a/src/crux/serve/metrics.go +++ b/src/crux/serve/metrics.go @@ -26,6 +26,7 @@ const ( requestDurationMetric = "crux.serve.request.duration" requestCountMetric = "crux.serve.requests" + authRejectionMetric = "crux.serve.auth.rejections" routeUnmatched = "unmatched" @@ -38,16 +39,28 @@ const ( methodOther = "_OTHER" ) +// Why authenticate refused a request: a fixed enum, so the rejection counter's +// reason attribute stays bounded. +const ( + rejectProjectForbidden = "project_forbidden" + rejectNoTokenConfigured = "no_token_configured" + rejectInvalidToken = "missing_or_invalid_token" +) + var ( + attrReason = attribute.Key("reason") attrMethod = attribute.Key("method") attrRoute = attribute.Key("route") attrStatusClass = attribute.Key("status_class") ) -// requestMetrics holds the two serve request instruments. +// requestMetrics holds the serve request and auth rejection instruments. type requestMetrics struct { duration metric.Float64Histogram requests metric.Int64Counter + // rejections counts requests authenticate refused with a 401. They never + // reach instrument, which sits inside authenticate. + rejections metric.Int64Counter } func newRequestMetrics(meter metric.Meter) requestMetrics { @@ -58,7 +71,9 @@ func newRequestMetrics(meter metric.Meter) requestMetrics { metric.WithDescription("Time crux serve took to answer a request.")) requests, _ := meter.Int64Counter(requestCountMetric, metric.WithDescription("crux serve requests, by method, route pattern and status class.")) - return requestMetrics{duration: duration, requests: requests} + rejections, _ := meter.Int64Counter(authRejectionMetric, + metric.WithDescription("crux serve requests refused by authentication, by reason.")) + return requestMetrics{duration: duration, requests: requests, rejections: rejections} } var globalRequestMetrics = sync.OnceValue(func() requestMetrics { @@ -168,6 +183,11 @@ func instrument(next http.Handler) http.Handler { }) } +// recordAuthRejection counts one 401 with only the bounded reason attribute. +func recordAuthRejection(ctx context.Context, reason string) { + globalRequestMetrics().rejections.Add(ctx, 1, metric.WithAttributes(attrReason.String(reason))) +} + func (m requestMetrics) record(ctx context.Context, d time.Duration, attrs []attribute.KeyValue) { opt := metric.WithAttributes(attrs...) m.duration.Record(ctx, d.Seconds(), opt) diff --git a/src/crux/serve/metrics_test.go b/src/crux/serve/metrics_test.go index 823724d53a..256fbeb7ef 100644 --- a/src/crux/serve/metrics_test.go +++ b/src/crux/serve/metrics_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/hivecommons/hive/crux/api" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/codes" sdkmetric "go.opentelemetry.io/otel/sdk/metric" @@ -334,3 +336,42 @@ func collectStatusClasses(t *testing.T, reader *sdkmetric.ManualReader) []string } return classes } + +func TestAuthRejectionsCountedByReason(t *testing.T) { + reader := useTestMetrics(t) + _, h := principalFixture(t) + + callWith(h, api.PathHealth, "wrong-secret-token") + callWith(h, api.PathHealth, "") + callWith(h, api.PathHealth, testProjectToken) + callWith(h, api.PathHealth, "another-wrong-token") + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + got := map[string]int64{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != authRejectionMetric { + continue + } + for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints { + if dp.Attributes.Len() != 1 { + t.Errorf("attributes = %v, want only reason", dp.Attributes) + } + reason, _ := dp.Attributes.Value(attrReason) + got[reason.AsString()] = dp.Value + } + } + } + want := map[string]int64{rejectInvalidToken: 3, rejectProjectForbidden: 1} + if len(got) != len(want) { + t.Fatalf("rejections = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("rejections[%s] = %d, want %d", k, got[k], v) + } + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 490eff2750..de7d07870a 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -209,6 +209,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) if project, projectToken := s.LaneAuth.Project(token); projectToken { if !s.isWorkService(r.URL.Path) { + recordAuthRejection(r.Context(), rejectProjectForbidden) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") return } @@ -217,6 +218,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { } if s.Token == "" { if network != networkUnix { + recordAuthRejection(r.Context(), rejectNoTokenConfigured) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") return } @@ -225,6 +227,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { } got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) if !ok || subtle.ConstantTimeCompare([]byte(got), []byte(s.Token)) != 1 { + recordAuthRejection(r.Context(), rejectInvalidToken) w.Header().Set("WWW-Authenticate", `Bearer realm="crux"`) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid bearer token") return From 0d1458e1a16df0b7461a93123af4f2189a8ff049 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 03:52:41 +0000 Subject: [PATCH 055/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-hub-reconcile-startup-probe.md | 1 - changelog.d/fixed-substrate-snapshot-on-idle-gc.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-hub-reconcile-startup-probe.md delete mode 100644 changelog.d/fixed-substrate-snapshot-on-idle-gc.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b761601f..ca2c0d1e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.1) + +### Fixed + +- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) +- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). + ## 2026-11-12 (v5.228.0) ### Added diff --git a/changelog.d/fixed-hub-reconcile-startup-probe.md b/changelog.d/fixed-hub-reconcile-startup-probe.md deleted file mode 100644 index 71a7a69c72..0000000000 --- a/changelog.d/fixed-hub-reconcile-startup-probe.md +++ /dev/null @@ -1 +0,0 @@ -- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) diff --git a/changelog.d/fixed-substrate-snapshot-on-idle-gc.md b/changelog.d/fixed-substrate-snapshot-on-idle-gc.md deleted file mode 100644 index 97a4d8f193..0000000000 --- a/changelog.d/fixed-substrate-snapshot-on-idle-gc.md +++ /dev/null @@ -1 +0,0 @@ -- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). From de8d38322bbfe1ee3503acd758df34437e6cd4fd Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:16:53 +0100 Subject: [PATCH 056/223] fix(hub): bound bearer token validation cache (#17846) Sweep expired entries at capacity and skip new entries when all slots remain occupied. Apply the same limit to accepted and rejected tokens. Hive-Run: hivecommons/hive#17844 Hive-Plan: issue-17844-cap-and-expired-sweep Hive-Spec: issue-17844#recommendation Signed-off-by: Karibhou Co-authored-by: Karibhou --- changelog.d/security-17844-bearer-cache.md | 1 + src/pkg/hub/saas_auth.go | 33 ++++++++--- src/pkg/hub/saas_auth_bearer_cache_test.go | 68 ++++++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 changelog.d/security-17844-bearer-cache.md diff --git a/changelog.d/security-17844-bearer-cache.md b/changelog.d/security-17844-bearer-cache.md new file mode 100644 index 0000000000..8d537d4e6e --- /dev/null +++ b/changelog.d/security-17844-bearer-cache.md @@ -0,0 +1 @@ +- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. diff --git a/src/pkg/hub/saas_auth.go b/src/pkg/hub/saas_auth.go index 5c0a7d5281..2daf88d43a 100644 --- a/src/pkg/hub/saas_auth.go +++ b/src/pkg/hub/saas_auth.go @@ -695,7 +695,8 @@ var ( ) const ( - ghTokenCacheTTL = 5 * time.Minute + ghTokenCacheMaxEntries = 10_000 + ghTokenCacheTTL = 5 * time.Minute // ghTokenNegativeTTL bounds how long a rejected bearer token (401/403, // including a rate-limited one) is remembered without re-probing // GET /user. Without it every request carrying a dead or exhausted token @@ -714,6 +715,28 @@ type ghTokenCacheEntry struct { expiresAt time.Time } +// cacheGHToken bounds both positive and negative entries. At capacity, expired +// entries are reclaimed; if all entries are live, new tokens are not cached. +func cacheGHToken(token string, entry ghTokenCacheEntry) { + ghTokenCacheMu.Lock() + defer ghTokenCacheMu.Unlock() + + if len(ghTokenCache) >= ghTokenCacheMaxEntries { + now := time.Now() + for key, cached := range ghTokenCache { + if !now.Before(cached.expiresAt) { + delete(ghTokenCache, key) + } + } + if len(ghTokenCache) >= ghTokenCacheMaxEntries { + if _, exists := ghTokenCache[token]; !exists { + return + } + } + } + ghTokenCache[token] = entry +} + // ghTokenValidateHTTPClient probes GET /user for bearer validation. It is a // plain client on purpose: the probe carries the CALLER's token, and routing // it through pkg/github's shared transport chain would let one exhausted user @@ -779,9 +802,7 @@ func (s *HubServer) validateGitHubToken(token string) string { hgithub.RecordRESTRequest(ghTokenValidateCaller, req.Method, req.URL.Path, resp.StatusCode, resp.Header) if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { - ghTokenCacheMu.Lock() - ghTokenCache[token] = ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())} - ghTokenCacheMu.Unlock() + cacheGHToken(token, ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())}) } return "" } @@ -792,9 +813,7 @@ func (s *HubServer) validateGitHubToken(token string) string { return "" } - ghTokenCacheMu.Lock() - ghTokenCache[token] = ghTokenCacheEntry{username: user.Login, expiresAt: time.Now().Add(ghTokenCacheTTL)} - ghTokenCacheMu.Unlock() + cacheGHToken(token, ghTokenCacheEntry{username: user.Login, expiresAt: time.Now().Add(ghTokenCacheTTL)}) return user.Login } diff --git a/src/pkg/hub/saas_auth_bearer_cache_test.go b/src/pkg/hub/saas_auth_bearer_cache_test.go index b9c62866e5..942b8ba717 100644 --- a/src/pkg/hub/saas_auth_bearer_cache_test.go +++ b/src/pkg/hub/saas_auth_bearer_cache_test.go @@ -68,6 +68,74 @@ func TestValidateGitHubToken_RejectedTokenIsNegativelyCached(t *testing.T) { } } +func TestValidateGitHubToken_CacheCapacityAndExpiry(t *testing.T) { + ghTokenCacheMu.Lock() + original := ghTokenCache + ghTokenCache = make(map[string]ghTokenCacheEntry) + ghTokenCacheMu.Unlock() + t.Cleanup(func() { + ghTokenCacheMu.Lock() + ghTokenCache = original + ghTokenCacheMu.Unlock() + }) + + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "Bearer valid-overflow" { + _ = json.NewEncoder(w).Encode(map[string]string{"login": "valid-user"}) + return + } + w.WriteHeader(http.StatusUnauthorized) + }) + srv := newHubServerForTest(t) + for i := range ghTokenCacheMaxEntries + 1 { + token := "rejected-capacity-" + strconv.Itoa(i) + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("rejected token resolved to %q", got) + } + } + ghTokenCacheMu.RLock() + size := len(ghTokenCache) + ghTokenCacheMu.RUnlock() + if size != ghTokenCacheMaxEntries { + t.Fatalf("cache has %d entries, want %d", size, ghTokenCacheMaxEntries) + } + + // Both rejected and accepted tokens still probe when the cache is full. + for range 2 { + if got := srv.validateGitHubToken("rejected-capacity-" + strconv.Itoa(ghTokenCacheMaxEntries)); got != "" { + t.Fatalf("overflow rejected token resolved to %q", got) + } + if got := srv.validateGitHubToken("valid-overflow"); got != "valid-user" { + t.Fatalf("overflow accepted token resolved to %q", got) + } + } + if got, want := atomic.LoadInt32(hits), int32(ghTokenCacheMaxEntries+5); got != want { + t.Fatalf("GET /user probed %d times, want %d", got, want) + } + ghTokenCacheMu.Lock() + entry := ghTokenCache["rejected-capacity-0"] + entry.expiresAt = time.Now().Add(-time.Second) + ghTokenCache["rejected-capacity-0"] = entry + ghTokenCacheMu.Unlock() + + // A full cache sweeps expired entries and makes room for a new positive entry. + for range 2 { + if got := srv.validateGitHubToken("valid-overflow"); got != "valid-user" { + t.Fatalf("accepted token after expiry resolved to %q", got) + } + } + if got, want := atomic.LoadInt32(hits), int32(ghTokenCacheMaxEntries+6); got != want { + t.Fatalf("GET /user probed %d times after expiry, want %d", got, want) + } + ghTokenCacheMu.RLock() + _, expiredPresent := ghTokenCache["rejected-capacity-0"] + size = len(ghTokenCache) + ghTokenCacheMu.RUnlock() + if expiredPresent || size != ghTokenCacheMaxEntries { + t.Fatalf("after sweep: expired present = %t, cache size = %d", expiredPresent, size) + } +} + // An expired negative entry probes again, and a token that has since become // valid resolves — the negative cache is a short hold, not a ban. func TestValidateGitHubToken_ExpiredNegativeEntryReprobes(t *testing.T) { From 4019513996b97b3ceab53106bb9465043e411072 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 04:23:04 +0000 Subject: [PATCH 057/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-crux-serve-auth-rejections.md | 1 - changelog.d/fixed-hub-bearer-negative-cache.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-crux-serve-auth-rejections.md delete mode 100644 changelog.d/fixed-hub-bearer-negative-cache.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ca2c0d1e0b..6d84059b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.2) + +### Fixed + +- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. +- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. + ## 2026-11-12 (v5.228.1) ### Fixed diff --git a/changelog.d/fixed-crux-serve-auth-rejections.md b/changelog.d/fixed-crux-serve-auth-rejections.md deleted file mode 100644 index ba4e30efba..0000000000 --- a/changelog.d/fixed-crux-serve-auth-rejections.md +++ /dev/null @@ -1 +0,0 @@ -- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. diff --git a/changelog.d/fixed-hub-bearer-negative-cache.md b/changelog.d/fixed-hub-bearer-negative-cache.md deleted file mode 100644 index 2fde88abb7..0000000000 --- a/changelog.d/fixed-hub-bearer-negative-cache.md +++ /dev/null @@ -1 +0,0 @@ -- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. From 8262a7b3ce7dff0e0318985b8e25d05a9b7787d7 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 04:42:17 +0000 Subject: [PATCH 058/223] =?UTF-8?q?=F0=9F=94=92=20knowledge:=20start=20the?= =?UTF-8?q?=20Plaud=20MCP=20server=20with=20an=20allowlisted=20env,=20not?= =?UTF-8?q?=20the=20hive=20process=20env=20(#17845)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stdioMCPDialer passed os.Environ() to a third-party MCP server fetched at run time (npx -y @plaud-ai/mcp@latest), so every hub-injected credential the hive process carries — DASHBOARD_AUTH_TOKEN (server-verified owner on the dashboard API), HIVE_TERMINAL_KEY, HIVE_HEARTBEAT_KEY, HIVE_HUB_SECRET, HIVE_UPSTREAM_PROXY_URL (proxy Basic credential), GitHub/Linear/provider keys — reached code the hive does not control. crux/agent strips the same set from agent panes (#17657); the MCP dialer bypassed that. The child now gets an allowlist: PATH, HOME, locale, TMPDIR, TERM, the proxy and CA-bundle variables Node needs to reach the registry through the egress proxy, NODE_*/npm_config_*/NPM_CONFIG_* and PLAUD_*. Any name that contains TOKEN, SECRET, PASSWORD or CREDENTIAL is dropped even when it matches a prefix. Fixes #17843 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-mcp-stdio-env-allowlist.md | 1 + src/docs/knowledge-connectors.md | 8 ++ src/pkg/knowledge/connector/mcpstdio.go | 57 +++++++++++++- src/pkg/knowledge/connector/mcpstdio_test.go | 76 +++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 changelog.d/security-mcp-stdio-env-allowlist.md diff --git a/changelog.d/security-mcp-stdio-env-allowlist.md b/changelog.d/security-mcp-stdio-env-allowlist.md new file mode 100644 index 0000000000..6d09e3f87a --- /dev/null +++ b/changelog.d/security-mcp-stdio-env-allowlist.md @@ -0,0 +1 @@ +- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index d9328667f9..cd39352198 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -256,6 +256,14 @@ stores or logs the token. By default the connector points the MCP server's so the token survives restarts; set `home: inherit` to use the hive process's HOME instead. +The MCP server is third-party code fetched at run time, so it does not inherit +the hive process environment. It is started with an allowlist only — `PATH`, +`HOME`, locale, `TMPDIR`, `TERM`, the `HTTP(S)_PROXY`/`NO_PROXY` and CA-bundle +variables, `NODE_*`/`npm_config_*` and `PLAUD_*` — and never the hub-injected +credentials (`DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY`, `HIVE_UPSTREAM_PROXY_URL`, +provider API keys). Pin `scope.command` to a specific `@plaud-ai/mcp@` +if you do not want a sync to pull a new upstream release unreviewed. + ### Setup 1. The hive host (or container image) needs **Node.js ≥ 20** so `npx` can run diff --git a/src/pkg/knowledge/connector/mcpstdio.go b/src/pkg/knowledge/connector/mcpstdio.go index 6cc36e2f15..5d916fcc72 100644 --- a/src/pkg/knowledge/connector/mcpstdio.go +++ b/src/pkg/knowledge/connector/mcpstdio.go @@ -266,9 +266,62 @@ func (b *lockedBuffer) Tail(n int) string { return s } +// mcpChildEnvNames are the exact environment variables a stdio MCP server +// inherits from the hive process; mcpChildEnvPrefixes extend the set by +// prefix. The server is third-party code fetched at run time (the Plaud +// default is `npx -y @plaud-ai/mcp@latest`), so it gets what a Node process +// needs to start and reach the registry through the egress proxy — never the +// hub-injected credentials the hive process carries (DASHBOARD_AUTH_TOKEN, +// HIVE_TERMINAL_KEY, HIVE_UPSTREAM_PROXY_URL, provider API keys, …), which +// crux/agent strips from agent panes for the same reason. +var mcpChildEnvNames = map[string]bool{ + "PATH": true, "HOME": true, "USER": true, "LOGNAME": true, "SHELL": true, + "TMPDIR": true, "TMP": true, "TEMP": true, "TZ": true, "TERM": true, "LANG": true, "LANGUAGE": true, + "HTTP_PROXY": true, "HTTPS_PROXY": true, "NO_PROXY": true, + "http_proxy": true, "https_proxy": true, "no_proxy": true, + "SSL_CERT_FILE": true, "SSL_CERT_DIR": true, "NODE_EXTRA_CA_CERTS": true, +} + +var mcpChildEnvPrefixes = []string{"LC_", "XDG_", "NODE_", "NPM_CONFIG_", "npm_config_", "PLAUD_"} + +// mcpChildEnv filters base (KEY=VALUE pairs) down to the allowlisted names. +// A name that looks like a credential is dropped even when it matches, so a +// future prefix cannot readmit one. +func mcpChildEnv(base []string) []string { + out := make([]string, 0, len(base)) + for _, entry := range base { + key, _, ok := strings.Cut(entry, "=") + if !ok || key == "" || mcpCredentialName(key) { + continue + } + if mcpChildEnvNames[key] { + out = append(out, entry) + continue + } + for _, p := range mcpChildEnvPrefixes { + if strings.HasPrefix(key, p) { + out = append(out, entry) + break + } + } + } + return out +} + +func mcpCredentialName(key string) bool { + u := strings.ToUpper(key) + for _, marker := range []string{"TOKEN", "SECRET", "PASSWORD", "CREDENTIAL"} { + if strings.Contains(u, marker) { + return true + } + } + return false +} + // stdioMCPDialer starts command as a child process per session. home, when // non-empty, overrides HOME so the server keeps its own token store in a -// persistent directory. +// persistent directory. The child's environment is the allowlisted subset +// mcpChildEnv keeps, not the hive process environment. func stdioMCPDialer(command, home string) mcpDialer { return func(ctx context.Context) (mcpClient, error) { argv, err := splitCommandLine(command) @@ -279,7 +332,7 @@ func stdioMCPDialer(command, home string) mcpDialer { return nil, errors.New("mcp: command is empty") } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) - cmd.Env = os.Environ() + cmd.Env = mcpChildEnv(os.Environ()) if home != "" { if err := os.MkdirAll(home, 0o700); err != nil { return nil, fmt.Errorf("mcp: creating home %s: %w", home, err) diff --git a/src/pkg/knowledge/connector/mcpstdio_test.go b/src/pkg/knowledge/connector/mcpstdio_test.go index 97a63e0df1..2fce7e8932 100644 --- a/src/pkg/knowledge/connector/mcpstdio_test.go +++ b/src/pkg/knowledge/connector/mcpstdio_test.go @@ -180,3 +180,79 @@ func TestStdioMCPDialerSpawnsProcess(t *testing.T) { t.Fatalf("HOME dir must be created: %v", err) } } + +func TestMCPChildEnvDropsHiveSecrets(t *testing.T) { + base := []string{ + "PATH=/usr/bin", + "HOME=/srv/hive", + "LC_ALL=C.UTF-8", + "NODE_OPTIONS=--max-old-space-size=512", + "npm_config_cache=/tmp/npm", + "HTTPS_PROXY=http://proxy:3128", + "NODE_EXTRA_CA_CERTS=/etc/hive/ca.pem", + "PLAUD_REGION=eu", + "DASHBOARD_AUTH_TOKEN=owner-token", + "HIVE_DASHBOARD_TOKEN=owner-token", + "HIVE_TERMINAL_KEY=terminal-key", + "HIVE_HEARTBEAT_KEY=heartbeat-key", + "HIVE_HUB_SECRET=hub-secret", + "HIVE_UPSTREAM_PROXY_URL=http://user:pass@corp:3128", + "HIVE_GITHUB_TOKEN=ghs_x", + "GITHUB_TOKEN=ghp_x", + "ANTHROPIC_API_KEY=sk-ant", + "LINEAR_CLIENT_SECRET=lin", + // Allowlisted prefix but credential-shaped: must still be dropped. + "NODE_AUTH_TOKEN=npm-token", + "PLAUD_SECRET=plaud", + "malformed-no-equals", + } + got := mcpChildEnv(base) + want := []string{ + "PATH=/usr/bin", + "HOME=/srv/hive", + "LC_ALL=C.UTF-8", + "NODE_OPTIONS=--max-old-space-size=512", + "npm_config_cache=/tmp/npm", + "HTTPS_PROXY=http://proxy:3128", + "NODE_EXTRA_CA_CERTS=/etc/hive/ca.pem", + "PLAUD_REGION=eu", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("mcpChildEnv =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// The spawned MCP server must not see the hub-injected credentials the hive +// process carries (hivecommons/hive#17843): the same secrets crux/agent +// strips from agent panes. +func TestStdioMCPDialerDoesNotLeakHiveEnv(t *testing.T) { + t.Setenv("DASHBOARD_AUTH_TOKEN", "dash_auth_leaked") + t.Setenv("HIVE_TERMINAL_KEY", "terminal_key_leaked") + t.Setenv("HIVE_UPSTREAM_PROXY_URL", "http://u:p@corp:3128") + t.Setenv("PLAUD_REGION", "eu") + + dir := t.TempDir() + script := filepath.Join(dir, "server.sh") + // Answers initialize, then reports the env it sees on the first tools/call. + body := "read line\necho '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}'\nread line\nread line\n" + + "printf '{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"dash=%s term=%s proxy=%s region=%s path_set=%s\"}]}}\\n' " + + "\"$DASHBOARD_AUTH_TOKEN\" \"$HIVE_TERMINAL_KEY\" \"$HIVE_UPSTREAM_PROXY_URL\" \"$PLAUD_REGION\" \"${PATH:+yes}\"\n" + if err := os.WriteFile(script, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + dial := stdioMCPDialer("sh "+script, filepath.Join(dir, "home")) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + c, err := dial(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer c.Close() + got, err := c.Call(ctx, "env", nil) + if err != nil { + t.Fatalf("call: %v", err) + } + if got != "dash= term= proxy= region=eu path_set=yes" { + t.Fatalf("child env = %q; hub secrets must be absent, PLAUD_* and PATH present", got) + } +} From 48a44303a19c2354ce170870bfd435e919023364 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 04:52:03 +0000 Subject: [PATCH 059/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.3 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/security-17844-bearer-cache.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/security-17844-bearer-cache.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d84059b50..6c86909221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.3) + +### Security + +- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. + ## 2026-11-12 (v5.228.2) ### Fixed diff --git a/changelog.d/security-17844-bearer-cache.md b/changelog.d/security-17844-bearer-cache.md deleted file mode 100644 index 8d537d4e6e..0000000000 --- a/changelog.d/security-17844-bearer-cache.md +++ /dev/null @@ -1 +0,0 @@ -- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. From c6ff24d4d6375ab7b465b5b82b5407aad6658c02 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 05:21:17 +0000 Subject: [PATCH 060/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.4 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/security-mcp-stdio-env-allowlist.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/security-mcp-stdio-env-allowlist.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c86909221..bf097b5b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.4) + +### Security + +- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). + ## 2026-11-12 (v5.228.3) ### Security diff --git a/changelog.d/security-mcp-stdio-env-allowlist.md b/changelog.d/security-mcp-stdio-env-allowlist.md deleted file mode 100644 index 6d09e3f87a..0000000000 --- a/changelog.d/security-mcp-stdio-env-allowlist.md +++ /dev/null @@ -1 +0,0 @@ -- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). From 53d891c3b91239d36e96f69983779f5e304262bb Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 22:38:25 -0700 Subject: [PATCH 061/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20omp=201?= =?UTF-8?q?8.18.0=20->=2018.19.0=20(#17850)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=18.19.0 SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 Signed-off-by: hive-release[bot] Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] --- src/Dockerfile | 6 +++--- src/Dockerfile.contributor | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 65c9b3a47d..70476b97bc 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -411,9 +411,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ # --- Layer 8.27: OMP CLI (Oh My Pi — backend: omp) --- # Mirrored from Dockerfile.contributor so hub agents can launch backend: omp. # Downloads are optional like Goose/agy, but checksum mismatches are fatal. -ARG OMP_VERSION=18.18.0 -ARG OMP_SHA256_AMD64=ec271e0d7df4c0f57acb6a7a28f8585d46bd050bcef8494f6d33db8a6c0dfbba -ARG OMP_SHA256_ARM64=b394bfbf7c0e0ba82e01913defb5fadf14a6ea5a71ee05348070ab5e40187144 +ARG OMP_VERSION=18.19.0 +ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 +ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64|x86_64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index be037588fc..0b4f73bf04 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -465,9 +465,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # No credential is baked in: omp reads the selected model's provider # credential from its documented environment variables (ANTHROPIC_API_KEY, # OPENAI_API_KEY, …), which the Justfile's container arm already forwards. -ARG OMP_VERSION=18.18.0 -ARG OMP_SHA256_AMD64=ec271e0d7df4c0f57acb6a7a28f8585d46bd050bcef8494f6d33db8a6c0dfbba -ARG OMP_SHA256_ARM64=b394bfbf7c0e0ba82e01913defb5fadf14a6ea5a71ee05348070ab5e40187144 +ARG OMP_VERSION=18.19.0 +ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 +ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ From ea886993f03a4628bbe44de5d1f327b0f4b40a98 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 22:52:26 -0700 Subject: [PATCH 062/223] =?UTF-8?q?=F0=9F=90=9B=20army:=20stop=20attributi?= =?UTF-8?q?ng=20adopted=20PRs=20to=20a=20remote=20agent's=20rework=20queue?= =?UTF-8?q?=20and=20show=20why=20a=20fix=20is=20not=20dispatched=20(#17853?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contribute fleet card listed every stage-store record by author in 072 or 074 as the relay's pending rework, so merged/closed PRs still carrying stage/074-adopted-by-hive (the store prunes them only after 14 days) and PRs the hive had taken over piled up as '33 PRs awaiting rework'. Only open, not-adopted, not-dispatched PRs count now: the forge cache decides open vs merged/closed when it knows the PR, otherwise a record the stage lane has not re-observed for an hour is treated as gone. A queued fix whose PR is gone leaves the fix queue instead of being handed to the relay. The idle-with-queue stall: a fix dispatched to a socket that dropped stays booked in flight with nobody holding it, so capacity 1 is busy forever and the relay gets neither the fix nor new work (task_unavailable fix_pr_in_flight, which the card hid behind 'idle · N awaiting rework'). The relay's next ready now releases such orphans (no attempt counted) and a task_complete for the dispatch on a reconnected socket settles it. The card prints the dispatcher's actual reason (pending_rework_blocked) instead of 'not yet dispatched', and links at most three PRs inline with the rest behind a show-N-more disclosure. Signed-off-by: Andrew Anderson --- changelog.d/fixed-army-rework-queue.md | 1 + src/docs/contributor-relay.md | 5 +- .../stage-labels.md | 3 +- src/pkg/dashboard/contribute_fixqueue.go | 67 +++++- src/pkg/dashboard/contribute_fleet.go | 24 ++- src/pkg/dashboard/contribute_fleet_rework.go | 174 +++++++++++++--- .../dashboard/contribute_fleet_rework_test.go | 190 +++++++++++++++++- src/pkg/dashboard/contribute_landing.go | 28 ++- src/pkg/dashboard/contribute_ws.go | 9 + src/pkg/fixqueue/fixqueue.go | 21 +- src/pkg/fixqueue/fixqueue_test.go | 25 +++ 11 files changed, 500 insertions(+), 47 deletions(-) create mode 100644 changelog.d/fixed-army-rework-queue.md diff --git a/changelog.d/fixed-army-rework-queue.md b/changelog.d/fixed-army-rework-queue.md new file mode 100644 index 0000000000..93eaad0192 --- /dev/null +++ b/changelog.d/fixed-army-rework-queue.md @@ -0,0 +1 @@ +- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 7f16ed2f28..c97d5a24cc 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1486,9 +1486,10 @@ Each row of the "Connected contributor agents (ClankeR)" list on `/contribute/op | `fix_mode` | `spawn` or `preempt`, the interrupt mode that capacity implies. | | `in_flight[]` | One entry per work item on the connection — the primary task plus every `fix-pr` sub-session — each with `kind` (`new` or `rework`), `task_kind`, `repo`, `number`, `title`, `url`, `stage` (the item's `stage/NNN-*` label as the hive last observed it), `mode` and `started_at`. The row shows one line per item with a **NEW** / **REWORK** badge. A relay running only a boundary-dispatched fix therefore shows as *working*, not *idle*. | | `fix_queue_depth` | Red PRs waiting in the contributor's server-side fix queue (not dispatched). | -| `pending_rework`, `pending_rework_prs[]` | The contributor's open PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. The row prints a red **N PRs awaiting rework (not yet dispatched)** line, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | +| `pending_rework`, `pending_rework_prs[]` | The contributor's **open** PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. Merged and closed PRs, PRs the hive adopted (074) and stage-store records the stage lane has stopped observing are never listed or counted. The row prints a red **N PRs awaiting rework (…)** line with at most three PRs linked inline and the rest behind a *show N more* disclosure, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | +| `pending_rework_blocked` | Why the pending rework is not on its way to the relay right now, in the dispatcher's own order: relay offline, no `fix_pr_interrupt_v1` capability, repository scope not acknowledged, not in this hub's fix queue, an earlier fix dispatch still booked on a dropped connection, retry backoff, focus window, slots busy, or simply waiting for the relay's next `ready`. Printed in place of the old *not yet dispatched*. Empty when nothing pends. | -`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections. A non-zero `pending_rework` on an idle row is the signal to look at — typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, or a contributor who is offline. +`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections. A non-zero `pending_rework` on an idle row is the signal to look at — `pending_rework_blocked` names the cause (typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, a fix dispatched to a socket that dropped before the relay reported on it, or a contributor who is offline). Whether a PR is still open comes from the forge cache the hive already holds; when the cache has no entry, a stage-store record the lane has not re-observed for an hour is treated as closed. A queued fix whose PR turns out merged or closed leaves the fix queue instead of being handed to the relay, and a fix booked in flight that no live connection holds is released at the relay's next `ready`. ## Reconnecting without losing in-flight work diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index a82ae1d89e..2dea077700 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -1256,7 +1256,8 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human- in-place adoption their push makes the hive yield (above). The contribute page's "PRs awaiting rework" list for a relay shows `adopting (contributor offline)` on a nudged PR whose relay connection is - stale, and `adopted by hive` once the PR is in 074. + stale; once the PR is in 074 it is the hive's and leaves that relay's + list. 5. Backpressure — the per-contributor fix budget. Capacity = the relay's advertised `max_parallel_fixes` (fix sub-sessions beside the primary), clamped by `contribute.max_parallel_fixes` (default 2); an unadvertised diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 787f819da1..bba745e1f2 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -476,6 +476,12 @@ func (h *ContributeWSHub) selectFixTask(c *ContributorConnection) *WSMessage { } user := c.profile.GitHubUsername sched := h.fixQueue() + // A `ready` says nothing is running on this relay. A fix dispatch the + // scheduler still books for this contributor that no live connection + // holds (its socket dropped before the relay reported on it) would + // otherwise occupy the only slot forever: nothing dispatched, new work + // refused with fix_pr_in_flight. Put it back at the head of the queue. + h.releaseOrphanedFixes(user, sched) if !sched.Pending(user) { return nil } @@ -496,13 +502,34 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin delete(c.fixTasksAt, taskID) } c.mu.Unlock() - if !ok || c.profile == nil { + if c.profile == nil { return ok } user := c.profile.GitHubUsername sched := h.fixQueue() + var prs []WSFixPR + if ok { + prs = assign.FixPRs + } else { + // The relay finished a fix it was handed on an earlier socket (the + // 1006 flap): this connection never saw the task_assign, but the + // scheduler still books the dispatch under this contributor. Settle + // it there rather than leaving the slot occupied for good. + for _, d := range sched.InFlight(user) { + if d.ID != taskID { + continue + } + ok = true + for _, it := range d.Items { + prs = append(prs, WSFixPR{Repo: it.Repo, Number: it.Number}) + } + } + if !ok { + return false + } + } if sched.CompleteDispatch(user, taskID) { - for _, pr := range assign.FixPRs { + for _, pr := range prs { h.forgetFixOwner(user, fixqueue.Key(pr.Repo, pr.Number)) } // Refill the freed slot with the next queued fix. @@ -513,6 +540,42 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin return true } +// heldFixTaskIDs returns the fix-pr task IDs every live connection of user +// currently holds. +func (h *ContributeWSHub) heldFixTaskIDs(user string) map[string]bool { + held := map[string]bool{} + h.mu.RLock() + defer h.mu.RUnlock() + for _, c := range h.connections { + c.mu.Lock() + if c.profile != nil && strings.EqualFold(c.profile.GitHubUsername, user) { + for id := range c.fixTasks { + held[id] = true + } + } + c.mu.Unlock() + } + return held +} + +// releaseOrphanedFixes returns to user's queue every dispatch the scheduler +// books in flight that no live connection holds. Called at a task boundary, +// when the relay has just said it is running nothing. +func (h *ContributeWSHub) releaseOrphanedFixes(user string, sched *fixqueue.Scheduler) { + var held map[string]bool + for _, d := range sched.InFlight(user) { + if held == nil { + held = h.heldFixTaskIDs(user) + } + if held[d.ID] { + continue + } + if items := sched.Release(user, d.ID); len(items) > 0 { + h.logger.Warn("[contribute-ws] fix-pr dispatch orphaned by a dropped connection; requeued", "username", user, "task", d.ID, "prs", len(items)) + } + } +} + // failFixTask settles a fix-pr task the relay reported failed: its PRs go // back in the queue with backoff, and a PR that has failed // fixTaskMaxAttempts times leaves the queue for the stage engine's re-nudge diff --git a/src/pkg/dashboard/contribute_fleet.go b/src/pkg/dashboard/contribute_fleet.go index 838d34f2f0..3b3465d68f 100644 --- a/src/pkg/dashboard/contribute_fleet.go +++ b/src/pkg/dashboard/contribute_fleet.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" "time" ) @@ -259,6 +260,23 @@ func (h *ContributeWSHub) FleetSnapshot() FleetSnapshot { Clankers: make([]FleetClanker, 0, len(h.connections)), Work: make([]FleetWorkItem, 0), } + // Fix-pr task IDs held per contributor across all their live sockets, + // gathered before any row lock is taken so no two connections are ever + // locked at once (fleetReworkFor names a dispatch nobody holds). + heldFix := map[string]map[string]bool{} + for _, c := range h.connections { + c.mu.Lock() + if c.profile != nil && len(c.fixTasks) > 0 { + user := strings.ToLower(c.profile.GitHubUsername) + if heldFix[user] == nil { + heldFix[user] = map[string]bool{} + } + for id := range c.fixTasks { + heldFix[user][id] = true + } + } + c.mu.Unlock() + } for _, c := range h.connections { c.mu.Lock() fc := FleetClanker{ @@ -352,7 +370,11 @@ func (h *ContributeWSHub) FleetSnapshot() FleetSnapshot { if task == nil { fc.IdleReason = c.lastIdleReason } - fc.FleetRework = h.fleetReworkFor(c, fc.Stale) + var held map[string]bool + if c.profile != nil { + held = heldFix[strings.ToLower(c.profile.GitHubUsername)] + } + fc.FleetRework = h.fleetReworkFor(c, fc.Stale, held) c.mu.Unlock() fc.CurrentTask = task fc.PromptPreview = promptPreview diff --git a/src/pkg/dashboard/contribute_fleet_rework.go b/src/pkg/dashboard/contribute_fleet_rework.go index cc44a499a2..7ed5d1d2cf 100644 --- a/src/pkg/dashboard/contribute_fleet_rework.go +++ b/src/pkg/dashboard/contribute_fleet_rework.go @@ -1,6 +1,7 @@ package dashboard import ( + "fmt" "sort" "strings" "time" @@ -51,8 +52,8 @@ type FleetInFlightItem struct { StartedAt string `json:"started_at,omitempty"` } -// FleetReworkPR is one of the contributor's PRs waiting in stage 072 that has -// not been dispatched to them. +// FleetReworkPR is one of the contributor's OPEN PRs waiting in stage 072 +// that has not been dispatched to them. type FleetReworkPR struct { Repo string `json:"repo"` Number int `json:"number"` @@ -63,16 +64,39 @@ type FleetReworkPR struct { Since string `json:"since,omitempty"` // Cause is the failing check / review ask, when the fix queue knows it. Cause string `json:"cause,omitempty"` - // State explains what the hive is doing about a PR the contributor is - // not working: "adopting (contributor offline)" while the connection is - // stale and the nudge went unanswered; "adopted by hive" once stage 074 - // took it over (hivecommons/hive#12436). + // State is "adopting (contributor offline)" while the connection is + // stale and the nudge went unanswered (hivecommons/hive#12436). A PR + // stage 074 already took over is not the contributor's any more and is + // not listed at all. State string `json:"state,omitempty"` } const ( FleetReworkStateAdopting = "adopting (contributor offline)" - FleetReworkStateAdopted = "adopted by hive" + // FleetReworkStateAdopted names the hand-off in logs and tests; such a + // PR no longer appears on the contributor's card. + FleetReworkStateAdopted = "adopted by hive" +) + +// fleetReworkSeenWithin bounds how long a stage-store record may go +// unobserved and still count as an open PR. The stage lane re-observes every +// open PR each sweep (about a minute); a record it has not touched for this +// long describes a PR that left the open listing (merged or closed) and is +// only waiting for the store's 14-day prune. +const fleetReworkSeenWithin = time.Hour + +// Why a queued fix is not on its way to the relay (FleetRework.PendingReworkBlocked). +const ( + FleetReworkBlockedOffline = "relay offline: no heartbeat within the presence window; the hive adopts the PR if the nudge stays unanswered" + FleetReworkBlockedNoFixPR = "relay does not accept fix-pr tasks (no fix_pr_interrupt_v1 capability); the PR only gets a GitHub comment" + FleetReworkBlockedScope = "relay's repository scope is not acknowledged (%s); the task boundary serves nothing until it is" + FleetReworkBlockedOrphan = "earlier fix dispatch %s is still booked in flight on a dropped connection; released at the relay's next ready" + FleetReworkBlockedBackoff = "retry backoff until %s after %d failed dispatch(es)" + FleetReworkBlockedFocus = "primary task protected by the focus window until %s; served at the next task boundary" + FleetReworkBlockedSlotsBusy = "all %d fix slot(s) busy" + FleetReworkBlockedAwaitReady = "waiting for the relay's next ready (fixes are served before new work)" + FleetReworkBlockedNotQueued = "not in this hub's fix queue; queued when the stage engine re-nudges" + fleetReworkBlockedIdleReasonFm = "%s; relay last reported idle: %s" ) // FleetRework is the per-contributor rework view spliced onto FleetClanker. @@ -88,14 +112,19 @@ type FleetRework struct { // InFlight lists every item this connection is working: the primary task // plus any fix-pr tasks running beside / instead of it. InFlight []FleetInFlightItem `json:"in_flight,omitempty"` - // FixQueueDepth is how many red PRs wait in the contributor's fix queue. + // FixQueueDepth is how many open red PRs wait in the contributor's fix queue. FixQueueDepth int `json:"fix_queue_depth"` - // PendingRework counts the contributor's PRs in + // PendingRework counts the contributor's OPEN PRs in // stage/072-returned-to-contributor that have NOT been dispatched to them - // (fix queue ∪ stage store, minus in-flight). Non-zero while the relay is - // idle is the operator's tell that rework is not reaching the agent. + // (fix queue ∪ stage store, minus in-flight, minus anything merged, + // closed, adopted by the hive or no longer observed by the stage lane). + // Non-zero while the relay is idle is the operator's tell that rework is + // not reaching the agent. PendingRework int `json:"pending_rework"` PendingReworkPRs []FleetReworkPR `json:"pending_rework_prs,omitempty"` + // PendingReworkBlocked says why the pending rework is not on its way to + // the relay right now (FleetReworkBlocked*); empty when nothing pends. + PendingReworkBlocked string `json:"pending_rework_blocked,omitempty"` } // stageStoreOrShared returns the injected stage store or the process-wide one. @@ -106,6 +135,44 @@ func (h *ContributeWSHub) stageStoreOrShared() *stagelabels.Store { return stagelabels.Shared() } +// prOpenState answers whether repo#number is still open from the forge +// cache the hive already holds (never a GitHub call): known=false when the +// cache has no entry. prOpenFn is the test seam. +func (h *ContributeWSHub) prOpenState(repo string, number int) (open, known bool) { + if h.prOpenFn != nil { + return h.prOpenFn(repo, number) + } + if h.server == nil || h.server.deps == nil || h.server.deps.GHClient == nil { + return false, false + } + fs := h.server.deps.GHClient.ForgeState() + if fs == nil { + return false, false + } + item, ok := fs.PR(repo, number) + if !ok { + return false, false + } + return item.Open(), true +} + +// reworkCandidateOpen reports whether a rework candidate is still an open +// PR of the contributor's: the forge cache decides when it knows the PR; +// otherwise a stage-store record the lane stopped observing means closed. +// A record stage 074 took over is the hive's now, not the contributor's. +func (h *ContributeWSHub) reworkCandidateOpen(repo string, number int, rec stagelabels.Record, hasRec bool, now time.Time) bool { + if hasRec && rec.Stage == stagelabels.AdoptedByHive { + return false + } + if open, known := h.prOpenState(repo, number); known { + return open + } + if hasRec && !rec.LastSeen.IsZero() && now.Sub(rec.LastSeen) > fleetReworkSeenWithin { + return false + } + return true +} + // isReworkStage reports whether a stage means "this PR is being repaired". func isReworkStage(st stagelabels.Stage) bool { switch st { @@ -140,9 +207,12 @@ func workKindFor(store *stagelabels.Store, taskKind, repo string, number int) (k // fleetReworkFor builds the rework view for one connection. Call with c.mu // held; it takes only the fix scheduler's and stage store's own locks. -func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) FleetRework { +// heldFixIDs are the fix-pr task IDs every live connection of this +// contributor holds (FleetSnapshot gathers them before locking c). +func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, heldFixIDs map[string]bool) FleetRework { var out FleetRework store := h.stageStoreOrShared() + now := time.Now() cap := h.fixCapacity(c) switch { case cap.Parallel: @@ -202,8 +272,6 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F return out } sched := h.fixQueue() - queued := sched.Queued(user) - out.FixQueueDepth = len(queued) // Dispatched on ANY of this contributor's connections counts as in flight. for _, d := range sched.InFlight(user) { for _, it := range d.Items { @@ -211,7 +279,16 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F } } pending := map[string]FleetReworkPR{} - for _, it := range queued { + for _, it := range sched.Queued(user) { + rec, hasRec := store.Get(stagelabels.Key(it.Repo, it.Number)) + if !h.reworkCandidateOpen(it.Repo, it.Number, rec, hasRec, now) { + // Merged, closed or adopted while queued: nothing left for the + // relay to fix, and handing it out would waste the agent's turn. + sched.Remove(user, it.Repo, it.Number) + h.forgetFixOwner(user, it.Key()) + continue + } + out.FixQueueDepth++ if inFlightKeys[it.Key()] { continue } @@ -236,6 +313,9 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F if repo == "" || number == 0 { continue } + if !h.reworkCandidateOpen(repo, number, ai.Record, true, now) { + continue + } pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label()} if !ai.Record.EnteredAt.IsZero() { pr.Since = ai.Record.EnteredAt.UTC().Format(time.RFC3339) @@ -245,20 +325,6 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F } pending[ai.Key] = pr } - for _, ai := range store.ByAuthor(user, stagelabels.AdoptedByHive) { - if _, dup := pending[ai.Key]; dup { - continue - } - repo, number := splitStageKey(ai.Key) - if repo == "" || number == 0 { - continue - } - pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label(), State: FleetReworkStateAdopted} - if !ai.Record.EnteredAt.IsZero() { - pr.Since = ai.Record.EnteredAt.UTC().Format(time.RFC3339) - } - pending[ai.Key] = pr - } keys := make([]string, 0, len(pending)) for k := range pending { keys = append(keys, k) @@ -268,9 +334,59 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F out.PendingReworkPRs = append(out.PendingReworkPRs, pending[k]) } out.PendingRework = len(out.PendingReworkPRs) + if out.PendingRework > 0 { + out.PendingReworkBlocked = h.fleetReworkBlockReason(c, stale, user, cap, sched, heldFixIDs, now) + } return out } +// fleetReworkBlockReason explains why user's pending rework is not on its +// way to the relay, in the order the dispatcher itself refuses. Call with +// c.mu held. +func (h *ContributeWSHub) fleetReworkBlockReason(c *ContributorConnection, stale bool, user string, cap fixqueue.Capacity, sched *fixqueue.Scheduler, heldFixIDs map[string]bool, now time.Time) string { + reason := func(r string) string { + if c.currentTask == nil && c.lastIdleReason != "" { + return fmt.Sprintf(fleetReworkBlockedIdleReasonFm, r, c.lastIdleReason) + } + return r + } + if stale { + return FleetReworkBlockedOffline + } + if !relaySupportsFixPR(c) { + return FleetReworkBlockedNoFixPR + } + if gate := repoScopeGateReasonLocked(c); gate != "" { + return reason(fmt.Sprintf(FleetReworkBlockedScope, gate)) + } + queued := sched.Queued(user) + if len(queued) == 0 { + return reason(FleetReworkBlockedNotQueued) + } + inflight := sched.InFlight(user) + for _, d := range inflight { + if !heldFixIDs[d.ID] { + return reason(fmt.Sprintf(FleetReworkBlockedOrphan, d.ID)) + } + } + if head := queued[0]; now.Before(head.NotBefore) { + return reason(fmt.Sprintf(FleetReworkBlockedBackoff, head.NotBefore.UTC().Format(time.RFC3339), head.Attempts)) + } + st := sched.Status(user, cap) + if c.currentTask != nil { + if st.FocusUntil != nil && now.Before(*st.FocusUntil) { + return fmt.Sprintf(FleetReworkBlockedFocus, st.FocusUntil.UTC().Format(time.RFC3339)) + } + if len(inflight) >= cap.Slots() { + return fmt.Sprintf(FleetReworkBlockedSlotsBusy, cap.Slots()) + } + } + if len(inflight) >= cap.BoundarySlots() { + return reason(fmt.Sprintf(FleetReworkBlockedSlotsBusy, cap.BoundarySlots())) + } + return reason(FleetReworkBlockedAwaitReady) +} + // splitStageKey inverts stagelabels.Key ("owner/repo#N"). func splitStageKey(key string) (repo string, number int) { i := strings.LastIndex(key, "#") diff --git a/src/pkg/dashboard/contribute_fleet_rework_test.go b/src/pkg/dashboard/contribute_fleet_rework_test.go index 704a383c41..f6038c07b9 100644 --- a/src/pkg/dashboard/contribute_fleet_rework_test.go +++ b/src/pkg/dashboard/contribute_fleet_rework_test.go @@ -3,6 +3,7 @@ package dashboard import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os/exec" @@ -266,6 +267,7 @@ func TestOpsClankerRowRendersCapacityAndRework(t *testing.T) { ".work-kind.new{", ".work-kind.rework{", ".clanker-idle-rework,.clanker-rework-pending{color:var(--status-error)", + ".clanker-rework-more>summary{", } { if !strings.Contains(body, want) { t.Errorf("operations page missing %q", want) @@ -273,7 +275,7 @@ func TestOpsClankerRowRendersCapacityAndRework(t *testing.T) { } script := ` const assert = require('node:assert/strict'); -var ccDeferredClankers = null, ccKnownClankers = {}, adminEnabled = false; +var ccDeferredClankers = null, ccKnownClankers = {}, adminEnabled = false, ccOpenPanes = {}, CLANKER_REWORK_INLINE_MAX = 3; const fleet = { querySelector() { return null; }, set innerHTML(v) { this.html = v; } }; const document = { getElementById(id) { return id === 'clanker-list' ? fleet : {}; } }; function ccUpdateArmy() {} @@ -327,6 +329,29 @@ assert.doesNotMatch(html, /idle: no matching work/, 'idle reason replaced while assert.match(html, /class="clanker-sub clanker-rework-pending" title="org\/r#2 \(build\), org\/r#3">2 PRs awaiting rework \(not yet dispatched\) · fix queue 1: org\/r#2<\/a>, org\/r#3<\/a>earlier fix dispatch fix-erin-1 is still booked[^<]*<\/span>\) · fix queue 1: org\/r#9<\/a>/, 'block reason'); +assert.doesNotMatch(html, /not yet dispatched/, 'generic wording gone when the hub explains'); + +// Five pending: three inline, the other two behind a closed disclosure. +var five = [1,2,3,4,5].map(function(n){return {repo: 'org/r', number: n, url: 'https://x/' + n};}); +renderClankers([{github_username: 'frank', fix_queue_depth: 5, pending_rework: 5, pending_rework_prs: five}]); +html = fleet.html; +assert.equal((html.match(/class="cc-issue-link work-ref"/g) || []).length, 5, 'every PR is still linked'); +var moreAt = html.indexOf('
show 2 more'); +assert.ok(moreAt > 0, 'disclosure present and closed'); +assert.ok(html.indexOf('org/r#3') < moreAt && html.indexOf('org/r#4') > moreAt, 'first three inline, rest behind the summary'); +ccOpenPanes['frank:rework'] = true; +renderClankers([{github_username: 'frank', fix_queue_depth: 5, pending_rework: 5, pending_rework_prs: five}]); +assert.match(fleet.html, /clanker-rework-more" data-pane="frank:rework" open>/, 'disclosure stays open across re-renders'); +delete ccOpenPanes['frank:rework']; +renderClankers([{github_username: 'gina', fix_queue_depth: 3, pending_rework: 3, pending_rework_prs: five.slice(0,3)}]); +assert.doesNotMatch(fleet.html, /clanker-rework-more/, 'three or fewer: no disclosure'); + // Idle, nothing pending: the pre-#12502 idle reason line is unchanged. renderClankers([{github_username: 'carol', idle_reason: 'no_matching_work', pending_rework: 0}]); html = fleet.html; @@ -359,14 +384,15 @@ func TestFleetReworkAdoptionState(t *testing.T) { store.NoteAuthor(stagelabels.Key("org/r", 3), "alice") fc := snapshotFor(t, hub, "alice") - if fc.Stale || len(fc.PendingReworkPRs) != 3 { + // The 074 PR is the hive's now: not listed, not counted. + if fc.Stale || fc.PendingRework != 2 || len(fc.PendingReworkPRs) != 2 { t.Fatalf("live: %+v", fc.FleetRework) } byNumber := map[int]FleetReworkPR{} for _, pr := range fc.PendingReworkPRs { byNumber[pr.Number] = pr } - if byNumber[1].State != "" || byNumber[2].State != "" || byNumber[3].State != FleetReworkStateAdopted || byNumber[3].Stage != stagelabels.AdoptedByHive.Label() { + if _, adopted := byNumber[3]; adopted || byNumber[1].State != "" || byNumber[2].State != "" { t.Fatalf("live states: %+v", fc.PendingReworkPRs) } @@ -381,10 +407,13 @@ func TestFleetReworkAdoptionState(t *testing.T) { for _, pr := range fc.PendingReworkPRs { byNumber[pr.Number] = pr } - // Nudged and unanswered → adopting; never nudged → plain 072; 074 → adopted. - if byNumber[1].State != FleetReworkStateAdopting || byNumber[2].State != "" || byNumber[3].State != FleetReworkStateAdopted { + // Nudged and unanswered → adopting; never nudged → plain 072; 074 → gone. + if _, adopted := byNumber[3]; adopted || byNumber[1].State != FleetReworkStateAdopting || byNumber[2].State != "" { t.Fatalf("stale states: %+v", fc.PendingReworkPRs) } + if fc.PendingReworkBlocked != FleetReworkBlockedOffline { + t.Fatalf("offline relay must say so: %q", fc.PendingReworkBlocked) + } raw, _ := json.Marshal(fc.PendingReworkPRs) if !strings.Contains(string(raw), `"state":"adopting (contributor offline)"`) { t.Fatalf("json: %s", raw) @@ -416,3 +445,154 @@ func TestContributeHubPresence(t *testing.T) { t.Fatalf("offline relay: %+v", p) } } + +// Only the contributor's OPEN PRs are their rework: a PR the forge cache +// reports merged or closed, or a stage-store record the lane stopped +// observing, is dropped from the count and (when queued) from the fix +// queue, so the relay is never handed a fix for a PR that is gone. +func TestFleetReworkDropsMergedClosedAndUnobserved(t *testing.T) { + hub, _, store := reworkHub(t) + now := time.Now() + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + + closed := map[string]bool{"org/r#2": true, "org/r#3": true} + hub.prOpenFn = func(repo string, number int) (bool, bool) { + k := fixqueue.Key(repo, number) + if closed[k] { + return false, true + } + if k == "org/r#1" { + return true, true + } + return false, false + } + for _, n := range []int{1, 2, 3, 4} { + store.Observe(stagelabels.Key("org/r", n), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", n), "alice") + } + // Queue #2 (closed) and #4 (unknown to the cache, fresh in the store). + hub.fixQueue().Enqueue("alice", fixqueue.Item{Repo: "org/r", Number: 2, Title: "closed"}) + hub.fixQueue().Enqueue("alice", fixqueue.Item{Repo: "org/r", Number: 4, Title: "open"}) + hub.rememberFixOwner("alice", "org/r#2") + hub.rememberFixOwner("alice", "org/r#4") + + fc := snapshotFor(t, hub, "alice") + got := map[int]bool{} + for _, pr := range fc.PendingReworkPRs { + got[pr.Number] = true + } + if fc.PendingRework != 2 || !got[1] || !got[4] || got[2] || got[3] { + t.Fatalf("pending: %+v", fc.FleetRework) + } + if fc.FixQueueDepth != 1 { + t.Fatalf("closed PR must leave the fix queue: %+v", fc.FleetRework) + } + if q := hub.fixQueue().Queued("alice"); len(q) != 1 || q[0].Number != 4 { + t.Fatalf("scheduler still holds the closed PR: %+v", q) + } + if fc.PendingReworkBlocked != FleetReworkBlockedAwaitReady { + t.Fatalf("idle relay with a queued fix waits for ready: %q", fc.PendingReworkBlocked) + } +} + +// A stage-store record the lane has not re-observed within +// fleetReworkSeenWithin describes a PR that left the open listing. +func TestFleetReworkDropsRecordTheLaneStoppedObserving(t *testing.T) { + hub, _, _ := reworkHub(t) + now := time.Now() + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + clock := now.Add(-2 * fleetReworkSeenWithin) + store := stagelabels.NewStore("", func() time.Time { return clock }) + hub.stageStore = store + store.Observe(stagelabels.Key("org/r", 1), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 1), "alice") + clock = now + store.Observe(stagelabels.Key("org/r", 2), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 2), "alice") + + fc := snapshotFor(t, hub, "alice") + if fc.PendingRework != 1 || fc.PendingReworkPRs[0].Number != 2 { + t.Fatalf("unobserved #1 must be dropped: %+v", fc.FleetRework) + } + // With no fix queue entry the hub can only wait for the engine's re-nudge. + if fc.PendingReworkBlocked != FleetReworkBlockedNotQueued { + t.Fatalf("block reason: %q", fc.PendingReworkBlocked) + } +} + +// A fix dispatched to a socket that then dropped stays booked in the +// scheduler with nobody holding it: capacity 1 is "busy" for good, so the +// relay is served nothing — not the fix, not new work. The snapshot names +// the orphan, and the relay's next `ready` releases and re-dispatches it. +func TestFleetReworkOrphanedDispatchIsNamedAndReleasedAtReady(t *testing.T) { + hub, _, store := reworkHub(t) + got := captureFixDispatch(hub) + now := time.Now() + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + store.Observe(stagelabels.Key("org/r", 1), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 1), "alice") + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("preempt dispatch expected, got %d", len(*got)) + } + orphanID := (*got)[0].TaskID + + // The socket that took the fix dies; the relay comes back on a new one. + hub.mu.Lock() + delete(hub.connections, c.connID) + hub.mu.Unlock() + c2 := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now.Add(time.Second)) + c2.lastIdleReason = taskUnavailableFixInFlight + + // Another red PR queues behind the orphan. + hub.Nudge(context.Background(), fixNudge("r", 2, "lint")) + fc := snapshotFor(t, hub, "alice") + if fc.PendingRework != 1 || fc.PendingReworkPRs[0].Number != 2 || len(fc.InFlight) != 0 { + t.Fatalf("pending: %+v", fc.FleetRework) + } + want := fmt.Sprintf(FleetReworkBlockedOrphan, orphanID) + "; relay last reported idle: " + taskUnavailableFixInFlight + if fc.PendingReworkBlocked != want { + t.Fatalf("block reason:\n got %q\nwant %q", fc.PendingReworkBlocked, want) + } + + // `ready` on the new socket: the orphan is released (no attempt counted) + // and the queue head goes out as a boundary fix. + msg := hub.selectFixTask(c2) + if msg == nil || msg.Kind != taskKindFixPR || msg.Mode != "boundary" { + t.Fatalf("boundary fix expected after release: %+v", msg) + } + if n := len(hub.fixQueue().InFlight("alice")); n != 1 { + t.Fatalf("exactly the new dispatch in flight, got %d", n) + } + for _, it := range hub.fixQueue().Queued("alice") { + if it.Attempts != 0 { + t.Fatalf("release must not count an attempt: %+v", it) + } + } +} + +// A relay that finished a fix handed out on an earlier socket reports +// task_complete on the new one; the scheduler settles it all the same. +func TestFleetReworkCompleteFixOnReconnectedSocket(t *testing.T) { + hub, _, _ := reworkHub(t) + got := captureFixDispatch(hub) + now := time.Now() + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("preempt dispatch expected, got %d", len(*got)) + } + hub.mu.Lock() + delete(hub.connections, c.connID) + hub.mu.Unlock() + c2 := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now.Add(time.Second)) + if !hub.completeFixTask(c2, (*got)[0].TaskID) { + t.Fatal("completion on the new socket must settle the scheduler's dispatch") + } + if st := hub.FixQueueStatus("alice"); st.InFlight != 0 || st.Queued != 0 { + t.Fatalf("dispatch should be settled: %+v", st) + } + if hub.completeFixTask(c2, "fix-alice-999") { + t.Fatal("an unknown task id is not a fix task") + } +} diff --git a/src/pkg/dashboard/contribute_landing.go b/src/pkg/dashboard/contribute_landing.go index c9fc370cb5..035b9511b4 100644 --- a/src/pkg/dashboard/contribute_landing.go +++ b/src/pkg/dashboard/contribute_landing.go @@ -472,6 +472,11 @@ code{background:var(--surface-0);padding:var(--sp-1) var(--sp-4);border-radius:v .work-kind.rework{background:color-mix(in srgb,var(--status-attention) var(--component-tint),transparent);color:var(--status-attention);border-color:color-mix(in srgb,var(--status-attention) var(--component-border),transparent)} .clanker-idle-rework,.clanker-rework-pending{color:var(--status-error);font-weight:600} .clanker-rework-pending .cc-issue-link{color:var(--status-error)} +.clanker-rework-pending .clanker-rework-why{font-weight:400;color:var(--text-muted)} +.clanker-rework-more{display:inline;font-weight:400} +.clanker-rework-more>summary{display:inline;cursor:pointer;color:var(--text-faint);user-select:none;list-style:none} +.clanker-rework-more>summary::-webkit-details-marker{display:none} +.clanker-rework-more[open]>summary{display:none} .clanker-pane{margin-top:var(--sp-3);font-size:.72rem} .clanker-pane>summary{cursor:pointer;color:var(--text-faint);font-family:ui-monospace,SFMono-Regular,Menlo,monospace;user-select:none;list-style:none} .clanker-pane>summary::before{content:"\25B8";display:inline-block;width:1em;color:var(--text-muted)} @@ -5518,14 +5523,29 @@ function clankerIdleLine(c){ if(pend>0)return '
idle · '+pend+' PR'+(pend===1?'':'s')+' awaiting rework
'; return c.idle_reason?('
idle: '+esc(idleReasonLabel(c.idle_reason))+'
'):''; } +// At most this many pending PRs are linked inline; the rest sit behind a +// "show N more" disclosure (the card is a status line, not a backlog). +var CLANKER_REWORK_INLINE_MAX=3; function clankerPendingReworkLine(c){ var pend=c.pending_rework|0; if(pend<=0)return ''; - var prs=(c.pending_rework_prs||[]).map(function(p){return (p.repo||'')+'#'+(p.number||'')+(p.cause?(' ('+p.cause+')'):'')+(p.state?(' — '+p.state):'');}).join(', '); - var links=(c.pending_rework_prs||[]).map(function(p){return clankerWorkRef(p)+(p.state?(' '+esc(p.state)+''):'');}).join(', '); + var items=c.pending_rework_prs||[]; + var prs=items.map(function(p){return (p.repo||'')+'#'+(p.number||'')+(p.cause?(' ('+p.cause+')'):'')+(p.state?(' — '+p.state):'');}).join(', '); + var refs=items.map(function(p){return clankerWorkRef(p)+(p.state?(' '+esc(p.state)+''):'');}); + var links=refs.slice(0,CLANKER_REWORK_INLINE_MAX).join(', '); + var rest=refs.slice(CLANKER_REWORK_INLINE_MAX); + if(rest.length){ + var key=(c.github_username||c.contributor_id||'').toLowerCase()+':rework'; + var open=ccOpenPanes[key]?' open':''; + links+=',
show '+rest.length+' more'+rest.join(', ')+'
'; + } var depth=c.fix_queue_depth|0; var q=depth>0?(' · fix queue '+depth):''; - return '
'+pend+' PR'+(pend===1?'':'s')+' awaiting rework (not yet dispatched)'+q+(links?(': '+links):'')+'
'; + // The server's own account of why the dispatcher is holding the fix + // (offline, no fix-pr capability, orphaned dispatch, backoff, focus + // window, waiting for the relay's next ready); older hubs send none. + var why=c.pending_rework_blocked?('not dispatched: '+esc(c.pending_rework_blocked)+''):'not yet dispatched'; + return '
'+pend+' PR'+(pend===1?'':'s')+' awaiting rework ('+why+')'+q+(links?(': '+links):'')+'
'; } // #7317 item 3 helpers. Both return '' when there is nothing to show so a // healthy row is byte-for-byte what it was before. @@ -5546,7 +5566,7 @@ function clankerPaneBlock(lines,key){ return '
agent pane · '+lines.length+' line'+(lines.length===1?'':'s')+'
'+esc(lines.join('\n'))+'
'; } onEl('clanker-list','toggle',function(e){ - var d=e.target;if(!d||!d.classList||!d.classList.contains('clanker-pane'))return; + var d=e.target;if(!d||!d.classList||!(d.classList.contains('clanker-pane')||d.classList.contains('clanker-rework-more')))return; var k=d.getAttribute('data-pane');if(!k)return; if(d.open)ccOpenPanes[k]=true;else delete ccOpenPanes[k]; },true); diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 87219cdb1f..f9c4e86eed 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -903,6 +903,9 @@ type ContributeWSHub struct { fixSched *fixqueue.Scheduler fixOwners map[string]map[string]bool fixDispatched func(*ContributorConnection, WSMessage) + // prOpenFn overrides the forge-cache lookup the fleet snapshot uses to + // drop merged/closed PRs from a contributor's rework; a test hook. + prOpenFn func(repo string, number int) (open, known bool) // fixFailedSig is closed (and replaced) whenever a relay reports a // fix-pr task failed, waking a Nudge that is waiting to learn whether its // dispatch was refused (hivecommons/hive#14668). fixAckGrace overrides @@ -3577,6 +3580,12 @@ func repoScopeGateReason(c *ContributorConnection) string { } c.mu.Lock() defer c.mu.Unlock() + return repoScopeGateReasonLocked(c) +} + +// repoScopeGateReasonLocked is repoScopeGateReason for a caller that already +// holds c.mu (the fleet snapshot explains an undispatched fix with it). +func repoScopeGateReasonLocked(c *ContributorConnection) string { if c.repoScopeInvalid { return taskUnavailableRepoScopeInvalid } diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index d0078def6d..863544914e 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -362,6 +362,19 @@ func (s *Scheduler) CompleteDispatch(user, id string) bool { // straight back to a relay that just refused it. It returns the requeued // items; nil when id is not in flight. func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { + return s.unwind(user, id, backoff, true, "requeue") +} + +// Release puts an in-flight dispatch back at the head of user's queue +// without counting an attempt or booking a backoff: the relay never reported +// on it (its socket dropped, or it asked for new work while the dispatch was +// still booked), so nothing about the PR itself failed. nil when id is not +// in flight. +func (s *Scheduler) Release(user, id string) []Item { + return s.unwind(user, id, 0, false, "release") +} + +func (s *Scheduler) unwind(user, id string, backoff time.Duration, countAttempt bool, event string) []Item { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] @@ -380,12 +393,14 @@ func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { out := d.public().Items notBefore := s.opts.Now().Add(backoff) for j := range out { - out[j].Attempts++ - out[j].NotBefore = notBefore + if countAttempt { + out[j].Attempts++ + out[j].NotBefore = notBefore + } c.queue = append(c.queue, out[j]) } sortQueue(c.queue) - s.log("requeue", user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) + s.log(event, user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) return out } return nil diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 94c3c5f7b6..42906a962e 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -418,3 +418,28 @@ func TestDispatchAt(t *testing.T) { t.Fatalf("inflight=%+v", got) } } + +// Release puts an in-flight dispatch back without an attempt or a backoff: +// the relay never reported on it, so the PR is immediately dispatchable. +func TestReleaseReturnsDispatchWithoutAttempt(t *testing.T) { + c := newClock() + s := newSched(c) + s.Enqueue("ida", red(1, time.Hour, c)) + d := s.Dispatch("ida", Capacity{Max: 1}) + if d == nil { + t.Fatal("dispatch expected") + } + if s.Release("ida", "nope") != nil || s.Release("zed", d.ID) != nil { + t.Fatal("unknown dispatch released") + } + got := s.Release("ida", d.ID) + if len(got) != 1 || got[0].Attempts != 0 || !got[0].NotBefore.IsZero() { + t.Fatalf("released = %+v", got) + } + if st := s.Status("ida", Capacity{Max: 1}); st.Queued != 1 || st.InFlight != 0 || st.FocusUntil != nil { + t.Fatalf("status after release = %+v", st) + } + if s.Dispatch("ida", Capacity{Max: 1}) == nil { + t.Fatal("released item must be dispatchable at once") + } +} From c02c4cc001b2a03b055ecfba0c7af0151e83ef4b Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:13:04 +0100 Subject: [PATCH 063/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20detect=20missing?= =?UTF-8?q?=20GitHub=20App=20repository=20coverage=20(#17733)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: detect missing GitHub App repository coverage Hive-Run: hivecommons/hive#17696 Hive-Plan: repository access and advisory recovery Hive-Spec: issue-17696#expected Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Signed-off-by: Karibhou * fix: add App repository coverage changelog Signed-off-by: Karibhou * fix: bound boot coverage and probe large installations Signed-off-by: Karibhou * docs: clarify GitHub App coverage polling Signed-off-by: Karibhou --------- Signed-off-by: Karibhou Co-authored-by: Karibhou Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-17696-app-repo-coverage.md | 1 + src/cmd/hive/boot_dashboard_api_deps.go | 15 +- src/cmd/hive/boot_dashboard_api_deps_test.go | 36 +++++ src/cmd/hive/boot_github_deps.go | 8 +- src/cmd/hive/boot_github_deps_test.go | 49 ++++++ src/cmd/hive/main.go | 33 ++++- src/cmd/hive/main_github_app.go | 16 ++ src/docs/github-app-setup.md | 27 +++- src/pkg/dashboard/api_governor.go | 75 +++++++--- .../dashboard/repo_check_access_probe_test.go | 139 ++++++++++++++++-- .../static-src/js/linear-repos-save.js | 19 +-- src/pkg/dashboard/static/index.html | 19 +-- src/pkg/github/advisory.go | 3 + src/pkg/github/advisory_retry_test.go | 81 ++++++++++ src/pkg/github/app_coverage.go | 48 ++++++ 15 files changed, 488 insertions(+), 81 deletions(-) create mode 100644 changelog.d/fixed-17696-app-repo-coverage.md create mode 100644 src/pkg/github/advisory_retry_test.go diff --git a/changelog.d/fixed-17696-app-repo-coverage.md b/changelog.d/fixed-17696-app-repo-coverage.md new file mode 100644 index 0000000000..8855f73672 --- /dev/null +++ b/changelog.d/fixed-17696-app-repo-coverage.md @@ -0,0 +1 @@ +- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). diff --git a/src/cmd/hive/boot_dashboard_api_deps.go b/src/cmd/hive/boot_dashboard_api_deps.go index 0d99bb23ee..b173ec14de 100644 --- a/src/cmd/hive/boot_dashboard_api_deps.go +++ b/src/cmd/hive/boot_dashboard_api_deps.go @@ -2,9 +2,11 @@ package main import ( "context" + "log/slog" "time" "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/github" ) // bootDashboardAPIDeps are the long-lived effects bootDashboardAPI performs @@ -22,13 +24,12 @@ import ( type bootDashboardAPIDeps struct { registerAPI func(srv *dashboard.Server, d *dashboard.Dependencies) setForgeAppInventory func(srv *dashboard.Server, fn func() dashboard.ForgeAppInventory) + classifyRepoCoverage func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) // startInstallDiscovery runs tryDiscover once immediately and then every // githubAppDiscoveryInterval until ctx is done. startInstallDiscovery func(ctx context.Context, tryDiscover func()) // startSelfHeal runs heal every githubAppSelfHealInterval until ctx is - // done. It does NOT run immediately: boot has just applied the classified - // App state and the first tick is cheap because heal no-ops while the - // banner is not showing. + // done. It does NOT run immediately: boot has just checked coverage. startSelfHeal func(ctx context.Context, heal func()) startInceptionWatcher func(ctx context.Context, w *dashboard.InceptionWatcher) } @@ -39,15 +40,13 @@ type bootDashboardAPIDeps struct { // requiring anyone to paste it. const githubAppDiscoveryInterval = 5 * time.Minute -// githubAppSelfHealInterval mirrors the heartbeat cadence so a stale banner -// clears within one heartbeat window of the app becoming healthy, without -// adding meaningful GitHub API load (the check only runs while the banner is -// actually showing). +// githubAppSelfHealInterval paces both coverage checks and banner healing. const githubAppSelfHealInterval = 2 * time.Minute func defaultBootDashboardAPIDeps() bootDashboardAPIDeps { return bootDashboardAPIDeps{ - registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { srv.RegisterAPI(d) }, + classifyRepoCoverage: classifyGitHubAppRepoCoverage, + registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { srv.RegisterAPI(d) }, setForgeAppInventory: func(srv *dashboard.Server, fn func() dashboard.ForgeAppInventory) { srv.SetForgeAppInventoryFn(fn) }, diff --git a/src/cmd/hive/boot_dashboard_api_deps_test.go b/src/cmd/hive/boot_dashboard_api_deps_test.go index 076ef3cdc0..bea7683d49 100644 --- a/src/cmd/hive/boot_dashboard_api_deps_test.go +++ b/src/cmd/hive/boot_dashboard_api_deps_test.go @@ -37,6 +37,9 @@ type bootDashboardAPIFake struct { func newBootDashboardAPIFake() *bootDashboardAPIFake { f := &bootDashboardAPIFake{} f.deps = bootDashboardAPIDeps{ + classifyRepoCoverage: func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) { + return false, "", github.AppStateUnknown + }, registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { f.registeredOn, f.registered = srv, d }, @@ -221,6 +224,39 @@ func TestBootDashboardAPIWith_TickBodiesAreCheapWhenNothingToDo(t *testing.T) { } } +func TestSelfHealDetectsCoverageWithoutExistingBanner(t *testing.T) { + for _, tc := range []struct { + name string + raise bool + state github.AppAuthState + }{ + {"missing", true, github.AppStateRepoNotCovered}, + {"transient", false, github.AppStateUnknown}, + {"covered", false, github.AppStateOK}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newBootDashboardAPIFake() + cfg := bootDashboardAPIConfig() + cfg.Project.PrimaryRepo = "private" + b := newBootDashboardAPIBoot(t, f, cfg) + b.ghClient = fakeGitHubClient(t) + called := false + f.deps.classifyRepoCoverage = func(_ context.Context, _ *github.AppAuth, org string, repos []string, _ *slog.Logger) (bool, string, github.AppAuthState) { + called = true + if org != "acme" || len(repos) != 3 || repos[2] != "private" { + t.Fatalf("coverage inputs: %q %v", org, repos) + } + return tc.raise, "select private in the installation", tc.state + } + b.bootDashboardAPIWith(f.deps) + f.healTick() + if !called || b.dashSrv.IsGitHubAppRequired() != tc.raise { + t.Fatalf("called=%v required=%v", called, b.dashSrv.IsGitHubAppRequired()) + } + }) + } +} + func TestBootDashboardAPIWith_ForgeInventoryResolvesTheActiveKey(t *testing.T) { f := newBootDashboardAPIFake() cfg := bootDashboardAPIConfig() diff --git a/src/cmd/hive/boot_github_deps.go b/src/cmd/hive/boot_github_deps.go index 1dbba48eb0..3f976f0c80 100644 --- a/src/cmd/hive/boot_github_deps.go +++ b/src/cmd/hive/boot_github_deps.go @@ -26,8 +26,9 @@ func defaultBootGitHubDeps() bootGitHubDeps { // notifier, ACMM inference, attribution hook, and on-disk default policy seed // run for real (cfg.Policies.LocalDir points a test at a temp dir). type bootAdvisoryDeps struct { - ensureAdvisoryIssue func(ctx context.Context, c *github.Client, repo string) (int, error) - classifyAppFailure func(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) + ensureAdvisoryIssue func(ctx context.Context, c *github.Client, repo string) (int, error) + classifyAppFailure func(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) + classifyRepoCoverage func(ctx context.Context, appAuth *github.AppAuth, org string, repos []string, logger *slog.Logger) (bool, string, github.AppAuthState) } func defaultBootAdvisoryDeps() bootAdvisoryDeps { @@ -35,6 +36,7 @@ func defaultBootAdvisoryDeps() bootAdvisoryDeps { ensureAdvisoryIssue: func(ctx context.Context, c *github.Client, repo string) (int, error) { return c.EnsureAdvisoryIssue(ctx, repo) }, - classifyAppFailure: classifyGitHubAppFailure, + classifyAppFailure: classifyGitHubAppFailure, + classifyRepoCoverage: classifyGitHubAppRepoCoverage, } } diff --git a/src/cmd/hive/boot_github_deps_test.go b/src/cmd/hive/boot_github_deps_test.go index 410e612bfa..1dc2faa30d 100644 --- a/src/cmd/hive/boot_github_deps_test.go +++ b/src/cmd/hive/boot_github_deps_test.go @@ -98,6 +98,9 @@ type bootAdvisoryFake struct { func newBootAdvisoryFake() *bootAdvisoryFake { f := &bootAdvisoryFake{ensureNum: 42} f.deps = bootAdvisoryDeps{ + classifyRepoCoverage: func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) { + return false, "", github.AppStateUnknown + }, ensureAdvisoryIssue: func(_ context.Context, _ *github.Client, repo string) (int, error) { f.ensured = append(f.ensured, repo) return f.ensureNum, f.ensureErr @@ -122,6 +125,52 @@ func bootAdvisoryConfig(t *testing.T) *config.Config { return cfg } +func TestBootAdvisoryDetectsMissingRepoCoverage(t *testing.T) { + for _, tc := range []struct { + name string + raise bool + state github.AppAuthState + }{ + {"missing", true, github.AppStateRepoNotCovered}, + {"transient", false, github.AppStateUnknown}, + {"covered", false, github.AppStateOK}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := bootAdvisoryConfig(t) + cfg.Project.PrimaryRepo = "private" + level := 0 + cfg.ACMMLevel = &level + b, _ := newDepsTestBoot(t, cfg) + b.ghClient = fakeGitHubClient(t) + f := newBootAdvisoryFake() + called := false + var coverageCtx context.Context + f.deps.classifyRepoCoverage = func(ctx context.Context, _ *github.AppAuth, org string, repos []string, _ *slog.Logger) (bool, string, github.AppAuthState) { + called = true + coverageCtx = ctx + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > advisoryEnsureBudget { + t.Fatalf("coverage deadline = %v, present=%v", deadline, ok) + } + if org != "acme" || len(repos) != 3 || repos[2] != "private" { + t.Fatalf("coverage inputs: %q %v", org, repos) + } + return tc.raise, "select private in the installation", tc.state + } + b.bootAdvisoryWith(f.deps) + if !called || b.githubAppRequired != tc.raise { + t.Fatalf("called=%v required=%v", called, b.githubAppRequired) + } + if coverageCtx.Err() != context.Canceled { + t.Fatalf("coverage context not canceled after check: %v", coverageCtx.Err()) + } + if tc.raise && (b.githubAppState != tc.state || b.githubAppDiag != "select private in the installation") { + t.Fatalf("coverage diagnosis: %v %q", b.githubAppState, b.githubAppDiag) + } + }) + } +} + func TestBootAdvisoryWithEnsuresIssueAndWritesPolicy(t *testing.T) { t.Setenv("HIVE_ADVISORY_ISSUE", "") cfg := bootAdvisoryConfig(t) diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index a7fca4a85b..51f69d747e 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1782,6 +1782,18 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { } } } + // Installation-level permissions do not establish that every configured + // repository is selected, including when advisory reads happen to succeed. + if b.ghClient != nil { + coverageCtx, cancelCoverage := context.WithTimeout(b.ctx, advisoryEnsureBudget) + raise, diag, state := deps.classifyRepoCoverage(coverageCtx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger) + cancelCoverage() + if raise { + b.githubAppRequired = true + b.githubAppDiag, b.githubAppState = diag, state + b.logger.Warn("GitHub App installation does not cover every configured repo", "state", state.String(), "detail", diag) + } + } b.advisoryStore = advisory.NewStore() @@ -3576,7 +3588,7 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { // as "app not installed / no read" — sending the operator after // credentials that were never broken. Checking first means the // specific, correct message wins over the generic one. - if raise, diag, state := classifyGitHubAppRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.cfg.Project.Repos, b.logger); raise { + if raise, diag, state := deps.classifyRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger); raise { b.dashSrv.SetGitHubAppPermIssue(diag) b.dashSrv.SetGitHubAppState(state.String()) b.logger.Warn("github app recheck: installation does not cover every configured repo", @@ -3665,12 +3677,9 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { // so a later transient write failure that re-set the flag was never // re-evaluated — the banner stuck until the pod was restarted. // - // The loop therefore runs for the lifetime of the process (it does NOT - // return after the first success) and, whenever the banner is currently - // showing, re-runs the SAME read+write verification as the manual "Re-check" - // button (githubAppRecheckFn, which calls diagnoseGitHubApp) and clears - // the flag on success. When the banner is not showing there is nothing to do, - // so the tick is a cheap no-op that makes no GitHub API calls. + // The loop runs for the lifetime of the process. It detects selection + // changes even when no banner is showing, and re-runs the manual re-check + // while a banner is present. Inconclusive coverage checks never raise one. { primaryRepo := b.cfg.Project.PrimaryRepo if primaryRepo == "" && len(b.cfg.Project.Repos) > 0 { @@ -3678,7 +3687,15 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { } if primaryRepo != "" { deps.startSelfHeal(b.ctx, func() { - // Nothing to heal unless the banner is showing. + if !b.dashSrv.IsGitHubAppRequired() && b.ghClient != nil { + if raise, diag, state := deps.classifyRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger); raise { + b.dashSrv.SetGitHubAppRequired(true) + b.dashSrv.SetGitHubAppPermIssue(diag) + b.dashSrv.SetGitHubAppState(state.String()) + b.logger.Warn("github app self-heal: installation does not cover every configured repo", "state", state.String(), "detail", diag) + return + } + } if !b.dashSrv.IsGitHubAppRequired() { return } diff --git a/src/cmd/hive/main_github_app.go b/src/cmd/hive/main_github_app.go index 5bb5fa99dd..b747bdca20 100644 --- a/src/cmd/hive/main_github_app.go +++ b/src/cmd/hive/main_github_app.go @@ -487,6 +487,22 @@ func classifyGitHubAppRepoCoverage(ctx context.Context, appAuth *github.AppAuth, return apphealth.ClassifyRepoCoverage(ctx, appAuth, org, repos, logger) } +// appCoverageRepos includes the primary repo even when it is not in the +// watched list; it is also used for advisory writes. +func appCoverageRepos(cfg *config.Config) []string { + repos := cfg.Project.Repos + primary := cfg.Project.PrimaryRepo + if primary == "" { + return repos + } + for _, repo := range repos { + if strings.EqualFold(repo, primary) || strings.EqualFold(repo, cfg.Project.Org+"/"+primary) || strings.EqualFold(cfg.Project.Org+"/"+repo, primary) { + return repos + } + } + return append(append([]string(nil), repos...), primary) +} + // The advisory digest posting policy moved to pkg/advisory (#7238 stage 2). // These wrappers keep the existing call sites unchanged; advisoryPostGate is // now an owned instance rather than a package-level struct tests reset. diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index b9b5794076..2976b95c7b 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -208,11 +208,12 @@ that has not approved a permission update is the one that still looks fine, and GitHub keeps an App on its old permissions until an org owner accepts, so a partly approved fleet is otherwise invisible. -Verdicts are computed where a GitHub call has failed and on the dashboard's -**Re-check** button, not on every eval cycle, so this adds no API calls to a -healthy hive. To read a specific installation's grants on demand — for example -to confirm the demo/canary install — press Re-check on that hive and read the -line above from its logs. +Permission verdicts are computed where a GitHub call has failed and on the +dashboard's **Re-check** button, not on every eval cycle. Repository coverage +is checked separately on the self-heal loop, including on a healthy hive, as +described below. To read a specific installation's grants on demand — for +example to confirm the demo/canary install — press Re-check on that hive and +read the line above from its logs. ## Install and configure Hive @@ -312,6 +313,22 @@ installation rather than creating a new one, so the `installation_id` in that redirect is one the hive already holds — the failed page does not mean App authentication is broken. +An installation can authenticate successfully while excluding a private +repository under **Only select repositories**. When adding a repo in the +dashboard, use the reported installation settings link to add it to the +selection, then re-check access. For repos already configured, the Forge App +banner identifies missing repository coverage; update the installation and +re-check it. A rejected advisory issue creation can then be retried without +waiting for the next UTC day. + +Boot coverage checks have a time limit; a timeout is inconclusive. The +background self-heal loop rechecks installation coverage every two minutes, +including on a healthy hive, using the cached installation token. Each check +walks up to 10 pages of `GET /installation/repositories`. If that bounded +listing is incomplete, the dashboard checks the repository's App installation +directly. Temporary GitHub errors still block adding the repository until a +check succeeds. + ## `/gh-setup` flow When the app's Setup URL points at `https:///gh-setup`, GitHub redirects back with `setup_action` and, for install/update, `installation_id`. diff --git a/src/pkg/dashboard/api_governor.go b/src/pkg/dashboard/api_governor.go index 096cb82a7e..6b5262c7d1 100644 --- a/src/pkg/dashboard/api_governor.go +++ b/src/pkg/dashboard/api_governor.go @@ -3366,20 +3366,6 @@ func firstRepoPathSegment(ref string) string { // a repo the operator is about to add, and — when it cannot — returns the // per-forge App install/authorize URL so the Repos tab can guide the user to // grant access before the repo is finally accepted (requirement #3). -// -// The check today probes at ORG granularity: DiscoverInstallationID(org) returns -// ErrNoInstallationForOrg when the App is not installed on the repo's org at all, -// which is the common "new org the App was never installed on" case and the one -// that hard-blocks reads and writes. On a positive result we report ok:true. -// -// TODO(repo-access-probe): tighten to REPO granularity. An App installed on the -// org "Only select repositories" may still lack THIS repo (see #2353's -// write-forbidden case). The deeper probe should mint an installation token and -// GET /repos/{org}/{repo} (s.deps.GHClient.GetRepo) — a 404/403 there means the -// installation exists but this repo is not in its selected set, which needs the -// same "manage installation" URL to add the repo to the selection. That probe is -// scoped out of this change to keep it focused; the org-level check plus the -// install workflow already covers the "App not installed on the org" footgun. func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Request) { var body struct { Repo string `json:"repo"` @@ -3420,6 +3406,18 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re jsonError(w, "cannot determine the org for this repo — use org/repo format", http.StatusBadRequest) return } + if len(parts) == 1 { + parts = []string{org, parts[0]} + } + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + jsonError(w, "cannot determine the repository — use org/repo format", http.StatusBadRequest) + return + } + repo := strings.TrimSuffix(parts[1], ".git") + if repo == "" { + jsonError(w, "cannot determine the repository — use org/repo format", http.StatusBadRequest) + return + } // An App-less hive (advisory-only, no private key) cannot and need not probe // installation access: it never writes. Treat it as "no access check needed" @@ -3433,13 +3431,13 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re if ctx == nil { ctx = r.Context() } - if _, err := s.deps.GHAppAuth.DiscoverInstallationID(ctx, org); err != nil { - // Not installed on this org (or discovery failed). Surface the correct - // per-forge install URL and arm the existing pending-install mechanism so - // the banner/recheck plumbing the App-required flow already uses drives - // this to completion. AppInstallURL() returns "" for a GHE host whose App - // slug was never configured — the UI renders that as a config message - // rather than a dead link. + installationID, err := s.deps.GHAppAuth.DiscoverInstallationID(ctx, org) + if err != nil { + if !errors.Is(err, github.ErrNoInstallationForOrg) { + s.logger.Warn("repo add: installation discovery failed", "org", org, "error", err) + jsonError(w, "could not check GitHub App installation access; please retry", http.StatusServiceUnavailable) + return + } installURL := "" if s.deps.Config != nil { installURL = s.deps.Config.GitHub.AppInstallURL() @@ -3457,6 +3455,41 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re }) return } + coverage, err := s.deps.GHAppAuth.InstallationCoverageForID(ctx, installationID) + if err != nil { + s.logger.Warn("repo add: installation repository coverage failed", "org", org, "repo", repo, "error", err) + jsonError(w, "could not check GitHub App repository access; please retry", http.StatusServiceUnavailable) + return + } + if _, found := coverage.Repos[strings.ToLower(org+"/"+repo)]; !found { + if coverage.Truncated { + covered, err := s.deps.GHAppAuth.RepositoryCoveredByInstallation(ctx, installationID, org, repo) + if err != nil { + s.logger.Warn("repo add: repository installation probe failed", "org", org, "repo", repo, "error", err) + jsonError(w, "could not check GitHub App repository access; please retry", http.StatusServiceUnavailable) + return + } + if covered { + okResponse(w, map[string]string{"status": "ok", "org": org}) + return + } + } + installURL := (github.AppAuthDiagnosis{ + ExpectedAccount: org, + InstallationID: installationID, + APIURL: s.deps.Config.GitHub.ResolvedAPIURL(), + }).InstallationSettingsURL() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "ok": false, + "needsInstall": true, + "org": org, + "installUrl": installURL, + "error": fmt.Sprintf("the Hive GitHub App installation cannot access %s/%s. Add the repository to the installation's repository access, then re-check.", org, repo), + }) + return + } okResponse(w, map[string]string{"status": "ok", "org": org}) } diff --git a/src/pkg/dashboard/repo_check_access_probe_test.go b/src/pkg/dashboard/repo_check_access_probe_test.go index 755d9047fc..3215153692 100644 --- a/src/pkg/dashboard/repo_check_access_probe_test.go +++ b/src/pkg/dashboard/repo_check_access_probe_test.go @@ -11,20 +11,14 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" ghpkg "github.com/hivecommons/hive/pkg/github" ) -// These tests cover the App-present half of handleGovernorRepoCheckAccess — -// the DiscoverInstallationID probe the endpoint exists for. The App-less, -// cross-forge, and no-org guards are covered in repo_add_form_test.go; here a -// real *ghpkg.AppAuth (generated RSA key) talks to a stub GitHub API so both -// probe outcomes run deterministically and offline: -// - installed → 200 {ok:true, status:"ok", org} -// - not installed → 409 {ok:false, needsInstall:true, org, installUrl} and -// the pending-install banner mechanism is armed. +// App-present access checks use a real AppAuth and an offline GitHub stub. // newDiscoveryAppAuth builds an AppAuth whose JWT generation succeeds and whose // API calls hit the given handler. Discovery results are memoised process-wide @@ -59,8 +53,7 @@ func newDiscoveryAppAuth(t *testing.T, handler http.HandlerFunc) *ghpkg.AppAuth return auth } -// installedOrgHandler answers GET /orgs/{org}/installation affirmatively for -// exactly one org and 404s everything else (including the listing fallback). +// installedOrgHandler supplies an installation covering widget. func installedOrgHandler(org string, id int64) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/orgs/"+org+"/installation" { @@ -71,6 +64,16 @@ func installedOrgHandler(org string, id int64) http.HandlerFunc { }) return } + if r.URL.Path == "/app/installations/"+strconv.FormatInt(id, 10)+"/access_tokens" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token":"installation-token","expires_at":"2099-01-01T00:00:00Z"}`)) + return + } + if r.URL.Path == "/installation/repositories" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"repositories": []any{map[string]any{"full_name": org + "/widget"}}}) + return + } if r.URL.Path == "/app/installations" { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`[]`)) @@ -130,6 +133,16 @@ func TestGovernorRepoCheckAccess_FullURLResolvesOrg(t *testing.T) { } } +func TestGovernorRepoCheckAccess_BareRepoUsesConfiguredOrg(t *testing.T) { + srv := newFullServer(t) + srv.deps.Config.Project.Org = "bare-org" + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, installedOrgHandler("bare-org", 123)) + w := checkAccessRequest(t, srv, `{"repo":"widget"}`) + if w.Code != http.StatusOK { + t.Fatalf("code = %d, want 200; body: %s", w.Code, w.Body.String()) + } +} + // App NOT installed on the org: 409 with needsInstall, the resolved org, the // per-forge install URL, and the pending-install banner mechanism armed so the // existing recheck plumbing drives the flow to completion. @@ -179,6 +192,112 @@ func TestGovernorRepoCheckAccess_NotInstalled409(t *testing.T) { } } +func TestGovernorRepoCheckAccess_RepoSelection(t *testing.T) { + for _, tc := range []struct { + name string + listStatus int + listBody string + wantStatus int + }{ + {"unselected", http.StatusOK, `{"repositories":[{"full_name":"selection-org/other"}]}`, http.StatusConflict}, + {"list failure", http.StatusInternalServerError, `{"message":"unavailable"}`, http.StatusServiceUnavailable}, + {"case insensitive", http.StatusOK, `{"repositories":[{"full_name":"SELECTION-ORG/WIDGET"}]}`, http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newFullServer(t) + base := installedOrgHandler("selection-org", 4242) + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/installation/repositories" { + if r.Header.Get("Authorization") != "Bearer installation-token" { + t.Errorf("repository listing did not use installation token") + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.listStatus) + _, _ = w.Write([]byte(tc.listBody)) + return + } + base(w, r) + }) + w := checkAccessRequest(t, srv, `{"repo":"selection-org/widget"}`) + if w.Code != tc.wantStatus { + t.Fatalf("code = %d, want %d; body: %s", w.Code, tc.wantStatus, w.Body.String()) + } + if tc.wantStatus == http.StatusConflict { + var resp struct { + NeedsInstall bool `json:"needsInstall"` + InstallURL string `json:"installUrl"` + Error string `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if !resp.NeedsInstall || !strings.Contains(resp.InstallURL, "/organizations/selection-org/settings/installations/4242") || !strings.Contains(resp.Error, "widget") { + t.Errorf("unexpected response: %+v", resp) + } + } + }) + } +} + +func TestGovernorRepoCheckAccess_DiscoveryFailure(t *testing.T) { + srv := newFullServer(t) + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unavailable", http.StatusInternalServerError) + }) + w := checkAccessRequest(t, srv, `{"repo":"discovery-failure/widget"}`) + if w.Code != http.StatusServiceUnavailable || srv.IsPendingGitHubAppInstall() { + t.Fatalf("unexpected response %d %s; pending=%v", w.Code, w.Body.String(), srv.IsPendingGitHubAppInstall()) + } +} + +func TestGovernorRepoCheckAccess_TruncatedCoverageUsesRepositoryInstallation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + wantStatus int + }{ + {"selected", http.StatusOK, `{"id":4242}`, http.StatusOK}, + {"not selected", http.StatusNotFound, `{"message":"Not Found"}`, http.StatusConflict}, + {"different installation", http.StatusOK, `{"id":2}`, http.StatusServiceUnavailable}, + {"missing installation id", http.StatusOK, `{}`, http.StatusServiceUnavailable}, + {"forbidden", http.StatusForbidden, `{"message":"rate limit"}`, http.StatusServiceUnavailable}, + {"rate limited", http.StatusTooManyRequests, `{"message":"rate limit"}`, http.StatusServiceUnavailable}, + {"server failure", http.StatusInternalServerError, `{"message":"unavailable"}`, http.StatusServiceUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newFullServer(t) + base := installedOrgHandler("truncated-org", 4242) + probed := false + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/installation/repositories": + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Link", `; rel="next"`) + _, _ = w.Write([]byte(`{"repositories":[]}`)) + case "/repos/truncated-org/widget/installation": + probed = true + if auth := r.Header.Get("Authorization"); !strings.HasPrefix(auth, "Bearer ") || auth == "Bearer installation-token" { + t.Errorf("repository installation probe did not use App JWT: %q", auth) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + default: + base(w, r) + } + }) + w := checkAccessRequest(t, srv, `{"repo":"truncated-org/widget"}`) + if !probed || w.Code != tc.wantStatus { + t.Fatalf("probed=%v code=%d, want %d; body: %s", probed, w.Code, tc.wantStatus, w.Body.String()) + } + if tc.wantStatus == http.StatusConflict && !strings.Contains(w.Body.String(), "/organizations/truncated-org/settings/installations/4242") { + t.Fatalf("missing installation settings link: %s", w.Body.String()) + } + }) + } +} + // Malformed body and blank repo are rejected before any probe runs. func TestGovernorRepoCheckAccess_BadInput(t *testing.T) { srv := newFullServer(t) diff --git a/src/pkg/dashboard/static-src/js/linear-repos-save.js b/src/pkg/dashboard/static-src/js/linear-repos-save.js index cd3a3852c8..5dc703eaa0 100644 --- a/src/pkg/dashboard/static-src/js/linear-repos-save.js +++ b/src/pkg/dashboard/static-src/js/linear-repos-save.js @@ -342,7 +342,7 @@ return; } - // Requirement #3: verify the App can access the repo's org. + // Verify the App installation covers this repository. setStatus('Checking GitHub App access…', 'var(--muted)'); try { const res = await fetch('/api/config/governor/repos/check-access', { @@ -351,14 +351,11 @@ }); const d = await res.json().catch(() => ({})); if (res.status === 409 && d && d.needsInstall) { - // App not installed on the new org: surface the correct per-forge - // install URL wired to the existing pending-install/recheck plumbing. - // Clicking Install flags the heartbeat (handleGitHubAppInstallClicked) - // exactly like the App-required banner; Re-check re-probes and, on - // success, completes the add. + // Guide the operator to install the App or update repository access, + // then re-check before adding the repo. const installBtn = (d.installUrl) - ? 'Install/authorize the App for ' + esc(d.org || '') + ' ↗' - : 'This hive\'s GitHub Enterprise App slug is not configured, so an install link cannot be built — ask your operator to install the Hive App for ' + esc(d.org || '') + '.'; + ? 'Manage App access for ' + esc(d.org || '') + ' ↗' + : 'An installation link cannot be built — ask your operator to manage the Hive App access for ' + esc(d.org || '') + '.'; setStatus('⚠ ' + esc(d.error || ('The App needs access to ' + val)) + '
' + installBtn + '  ·  Re-check & add', 'var(--amber)'); return; } @@ -367,11 +364,7 @@ return; } } catch (e) { - // Network error probing access is non-fatal: the server re-validates the - // single-host rule and the default-repo rule on save, so let the add - // proceed rather than hard-block on a transient dashboard hiccup. - _appendRepoAccepted(val); - setStatus('Added (App-access check could not be reached; the server will re-validate on save).', 'var(--muted)'); + setStatus('✕ Could not check GitHub App access. Please retry.', 'var(--red)'); return; } _appendRepoAccepted(val); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 0b3097ddd5..2917b2797f 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -71539,7 +71539,7 @@

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'} ({})); if (res.status === 409 && d && d.needsInstall) { - // App not installed on the new org: surface the correct per-forge - // install URL wired to the existing pending-install/recheck plumbing. - // Clicking Install flags the heartbeat (handleGitHubAppInstallClicked) - // exactly like the App-required banner; Re-check re-probes and, on - // success, completes the add. + // Guide the operator to install the App or update repository access, + // then re-check before adding the repo. const installBtn = (d.installUrl) - ? 'Install/authorize the App for ' + esc(d.org || '') + ' ↗' - : 'This hive\'s GitHub Enterprise App slug is not configured, so an install link cannot be built — ask your operator to install the Hive App for ' + esc(d.org || '') + '.'; + ? 'Manage App access for ' + esc(d.org || '') + ' ↗' + : 'An installation link cannot be built — ask your operator to manage the Hive App access for ' + esc(d.org || '') + '.'; setStatus('⚠ ' + esc(d.error || ('The App needs access to ' + val)) + '
' + installBtn + '  ·  Re-check & add', 'var(--amber)'); return; } @@ -71564,11 +71561,7 @@

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}

Date: Thu, 12 Nov 2026 06:24:44 +0000 Subject: [PATCH 064/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.5 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-army-rework-queue.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-army-rework-queue.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bf097b5b21..da7895396c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.5) + +### Fixed + +- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. + ## 2026-11-12 (v5.228.4) ### Security diff --git a/changelog.d/fixed-army-rework-queue.md b/changelog.d/fixed-army-rework-queue.md deleted file mode 100644 index 93eaad0192..0000000000 --- a/changelog.d/fixed-army-rework-queue.md +++ /dev/null @@ -1 +0,0 @@ -- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. From 835c1cdb0f9343601d796c868cb1dfeb53f1fc4a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 23:45:01 -0700 Subject: [PATCH 065/223] =?UTF-8?q?=E2=9C=A8=20knowledge:=20Wispr=20Flow?= =?UTF-8?q?=20dictation=20history=20as=20a=20knowledge=20source=20(#17855)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: Wispr Flow dictation history as a knowledge source Adds knowledge.connectors[].type: wisprflow. Reads the desktop app's local flow.sqlite read-only through the sqlite3 CLI (text columns only, never the audio/screenshot BLOBs) or a folder of exported .json/.ndjson/.csv/.md files, and emits one fact per finalised dictation (or per day) with the dictation time, source app and URL. Incremental by timestamp with a 2h overlap; min_words and apps filters; tags. Dashboard: Knowledge -> Integrations card (path, status, last sync, count), Add-source entry pointing out that new dictations need no connector at all (dictate into + New Fact). Docs section next to Plaud's; changelog fragment. Fixes #17854 Signed-off-by: Andrew Anderson * 🐛 knowledge: wisprflow tests run without the sqlite3 CLI Sync tests now use a fake sqlite3 runner fed from testdata/wisprflow/history.json so the SQLite path is exercised (and counted for coverage) on runners without the CLI; the real-CLI test skips when sqlite3 is absent. TestWisprValidate passes an explicit path so it no longer depends on a macOS default location. Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../added-knowledge-wisprflow-source.md | 1 + src/docs/knowledge-connectors.md | 97 ++- .../knowledge_integrations_modal_test.go | 42 ++ .../dashboard/static-src/js/config-state.js | 1 + .../dashboard/static-src/js/config-tabs.js | 9 +- src/pkg/dashboard/static-src/js/knowledge.js | 16 + src/pkg/dashboard/static/index.html | 26 +- src/pkg/knowledge/connector/connector.go | 3 +- src/pkg/knowledge/connector/connector_test.go | 2 +- .../export/2026-11-11-1430-cursor.md | 3 + .../wisprflow/export/2026-11-11.ndjson | 3 + .../testdata/wisprflow/export/dictations.csv | 2 + .../testdata/wisprflow/export/dictations.json | 4 + .../testdata/wisprflow/export/ignored.bin | 1 + .../wisprflow/export/nested/frontmatter.md | 6 + .../connector/testdata/wisprflow/history.json | 6 + .../connector/testdata/wisprflow/history.sql | 14 + src/pkg/knowledge/connector/wisprflow.go | 577 ++++++++++++++++++ .../knowledge/connector/wisprflow_export.go | 259 ++++++++ src/pkg/knowledge/connector/wisprflow_test.go | 350 +++++++++++ 20 files changed, 1412 insertions(+), 10 deletions(-) create mode 100644 changelog.d/added-knowledge-wisprflow-source.md create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/history.json create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/history.sql create mode 100644 src/pkg/knowledge/connector/wisprflow.go create mode 100644 src/pkg/knowledge/connector/wisprflow_export.go create mode 100644 src/pkg/knowledge/connector/wisprflow_test.go diff --git a/changelog.d/added-knowledge-wisprflow-source.md b/changelog.d/added-knowledge-wisprflow-source.md new file mode 100644 index 0000000000..3e63a6018a --- /dev/null +++ b/changelog.d/added-knowledge-wisprflow-source.md @@ -0,0 +1 @@ +- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index cd39352198..6f84bf0979 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -1,6 +1,6 @@ # Knowledge connectors -External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis and Plaud voice notes) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. +External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis, Plaud voice notes and Wispr Flow dictation) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. For the shared configuration fields, the fact format and how connectors relate to `git_sources` and `documents`, see [knowledge-curator.md → Connectors](knowledge-curator.md#connectors-knowledgeconnectors). @@ -10,7 +10,7 @@ For the shared configuration fields, the fact format and how connectors relate t knowledge: connectors: - name: team-wiki # lowercase letters, digits, dashes - type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud + type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud | wisprflow enabled: true interval: 15m # minimum 1m; blank = 15m layer: project # personal | project | org | community @@ -311,6 +311,99 @@ if you do not want a sync to pull a new upstream release unreviewed. `document` connector (`file_path`). No code, still $0, and no MCP sign-in. - Everything in this section costs **$0** beyond your Plaud subscription. +## Wispr Flow (dictation, `type: wisprflow`) + +[Wispr Flow](https://wisprflow.ai) is dictation into whatever field has focus: +hold the Flow key, speak, release, and polished text is pasted where the +cursor is. That gives two $0 paths into the hive. + +**Direct dictation — no connector.** Put the cursor in **Knowledge → + New +Fact** (title or body), the Import box, an agent prompt, or any other text +field in the dashboard and dictate. Nothing to configure. This is the +recommended way to get *new* thoughts in. + +**History connector.** The desktop app keeps every dictation in a local +SQLite file. The `wisprflow` connector reads it read-only and turns each +dictation (what you said, when, and into which app) into a fact, so ideas you +dictated into Slack or your editor last week are searchable by agents too. +There is no official export API for dictation history — Wispr's remote MCP +server covers Notetaker meetings only and states it cannot read dictation +history, and the Voice Interface API is invite-only and does speech-to-text, +not history — so the local file is the only free source. The schema is +community-documented (`History` table: `transcriptEntityId`, `asrText`, +`formattedText`, `editedText`, `timestamp`, `app`, `url`, `duration`, +`numWords`, `status`, `isArchived`) and can change with an app update. + +```yaml +knowledge: + connectors: + - name: wispr + type: wisprflow + layer: personal + interval: 15m + scope: + # path: /Users/me/Library/Application Support/Wispr Flow/flow.sqlite # default on macOS; %APPDATA%\Wispr Flow\flow.sqlite on Windows + # path: /data/knowledge/wispr-export # or a folder of exported .json/.ndjson/.csv/.md files + since_days: "30" # how far back the first sync reaches + max_items: "500" # dictations read per sync + min_words: "3" # skip "ok", "yes", … + group: dictation # dictation = one fact each; day = one fact per calendar day + apps: "" # optional allow-list matched against the app bundle id, e.g. "slack, obsidian, vscode" + include_raw: "false" # also keep the raw ASR text under "Raw transcript" + tags: "voice, ideas" # extra tags written as attr_tags + # sqlite3: sqlite3 # CLI used to read the file; ships with macOS and the hive image +``` + +No `auth` block and no sign-in: the data is already on disk. The connector +shells out to the `sqlite3` CLI with `-readonly -json` and selects text +columns only (the table also holds audio and screenshot BLOBs, which are never +read). Rows with `status = formatted` and `isArchived = 0` are used; +`editedText` wins over `formattedText` over `asrText`. + +### Setup + +1. **Same machine as the hive (laptop hive):** add the connector from + **Knowledge → 🔌 Integrations → Wispr Flow dictation** and leave `path` + empty. On macOS the hive process may need **Full Disk Access** (System + Settings → Privacy & Security) to read `~/Library/Application Support`. +2. **Hive on a cluster or another host:** sync the file or an export to a + path the hive can read and set `path` to it. Any of these work: + - copy `flow.sqlite` (or one of the app's timestamped `backups/*.sqlite`) + into the knowledge PVC, e.g. with a cron `kubectl cp`; + - run a community exporter such as `wispr export -o ` + ([iamnikolie/wispr-cli](https://github.com/iamnikolie/wispr-cli), writes + `dictations/*.md`) or + [junxit/wispr-flow-exporter](https://github.com/junxit/wispr-flow-exporter) + (Markdown/JSON/NDJSON) and point `path` at the folder, or commit that + folder to a Git-synced vault and let the existing `git` connector pick + it up. +3. Click **Sync now**. New dictations are picked up every `interval`. + +Export folders are walked recursively; `.json` (array or +`{"dictations": [...]}`), `.ndjson`/`.jsonl`, `.csv` (header row) and `.md` +(one dictation per file; optional front matter `id:`, `timestamp:`, `app:`, +otherwise the date in the filename or the file mtime) are read, other files +are ignored. A file that fails to parse is reported as a partial sync and the +rest are still imported. + +### Behaviour + +- Incremental by `timestamp`: each sync re-reads from the newest dictation + seen minus a 2 h overlap (Flow updates a row in place as it finalises), or + `since_days` back on the first run. With `group: day` the current day's + fact is rebuilt whole. +- Facts carry `source: wisprflow`, `attr_wispr_transcript_id`, + `attr_dictated_at`, `attr_app` (bundle id), `attr_num_words`, + `attr_language` and the dictation date as the vault path + (`Wispr Flow/2026-11-11`). The title is the first ten words. +- The listing is incremental, so dictations you archive or delete in Flow + are not tombstoned automatically; remove the fact file or the connector. +- Everything here costs **$0** beyond your Wispr Flow plan; the free tier's + weekly word cap applies to dictating, not to reading history. Tip: add + your project vocabulary (`hive`, `dibs`, `spek`, `ACMM`, `crux`, repo + names) under Flow → Settings → Dictionary so it is spelled right the first + time. + ## Notion (`type: notion`) The Notion connector syncs pages and database rows through the public Notion diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 0e79b35a80..2c3cb41497 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -184,3 +184,45 @@ func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { } } } + +// TestKnowledgeIntegrationsModalWisprFlowCard guards the Wispr Flow entry: +// it is a plain local-file connector (no sign-in buttons), the card shows +// the configured path, the Knowledge pane links to it and documents the +// direct-dictation path, and no browser-native dialog is used. +func TestKnowledgeIntegrationsModalWisprFlowCard(t *testing.T) { + b, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatalf("reading embedded static/index.html: %v", err) + } + html := string(b) + for _, want := range []string{ + `'wisprflow': { label: 'Wispr Flow dictation'`, + `'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)'`, + `['group', 'Fact per dictation or day', 'dictation']`, + `if (type === 'wisprflow') html += kbWisprFlowSourceLine(c);`, + `data-kb-wisprflow-note`, + `data-action="kbOpenWisprFlowIntegration"`, + `function kbWisprFlowSourceLine(c)`, + `function kbOpenWisprFlowIntegration()`, + `Wispr Flow dictation`, + `+ New Fact and hold the Flow key`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Wispr Flow card snippet %q", want) + } + } + if !strings.Contains(html, `'sharepoint', 'wisprflow']`) { + t.Error("wisprflow missing from KB_CONNECTOR_TYPE_ORDER") + } + start := strings.Index(html, "function kbWisprFlowSourceLine(c)") + end := strings.Index(html, "function kbOpenWisprFlowIntegration()") + if start < 0 || end < start { + t.Fatal("Wispr Flow block not found") + } + block := html[start:end] + for _, native := range []string{"window.confirm(", "window.alert(", " confirm(", " alert(", "prompt("} { + if strings.Contains(block, native) { + t.Errorf("Wispr Flow block uses a browser-native dialog %q", native) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 459c8a898a..162cf7e258 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -128,6 +128,7 @@ { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Wispr Flow dictation connector', key: 'knowledge.connectors', lazy: true, text: 'wispr flow wisprflow dictation voice history flow.sqlite sqlite export path since_days max_items min_words group apps include_raw tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index fbd7bfa2ef..03d69b03f3 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -289,7 +289,7 @@ } // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint', 'wisprflow']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -300,7 +300,8 @@ 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, - 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } + 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' }, + 'wisprflow': { label: 'Wispr Flow dictation', description: 'Sync your Wispr Flow dictation history (what you spoke, when, into which app) from the desktop app\'s local flow.sqlite or an export folder. Read-only, no sign-in, $0 beyond your Wispr Flow plan.' } }; const KC_TYPE_ICONS = { // Vendor marks: simple-icons (CC0 1.0) paths, 24x24 viewBox, rendered with currentColor. @@ -315,6 +316,7 @@ 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', + 'wisprflow': 'M3 10h2v4H3zm4-3h2v10H7zm4-5h2v20h-2zm4 5h2v10h-2zm4 3h2v4h-2z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -332,7 +334,8 @@ 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], - 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] + 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']], + 'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)', '/Users/me/Library/Application Support/Wispr Flow/flow.sqlite'], ['since_days', 'Days back on first sync', '30'], ['max_items', 'Max dictations per sync', '500'], ['min_words', 'Skip dictations shorter than (words)', '3'], ['group', 'Fact per dictation or day', 'dictation'], ['apps', 'Only these apps (comma-separated, optional)', 'slack, obsidian, vscode'], ['include_raw', 'Include raw ASR text', 'false'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['sqlite3', 'sqlite3 command (optional)', 'sqlite3']] }; let _kcState = { loaded: false, error: '', connectors: [], rows: [], types: [], layers: ['personal', 'project', 'org', 'community'], runtime: false, publish: null }; diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index eb322b6c55..81daf98c72 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -1981,6 +1981,12 @@ html += '
'; html += '
'; + html += '
🗣️ Wispr Flow dictation'; + html += '
'; + html += '
Dictate straight into any field here — put the cursor in + New Fact and hold the Flow key; no connector needed. To also pull your past dictations in, the Wispr Flow connector reads the app\'s local history (flow.sqlite or an export folder). $0 beyond your Wispr Flow plan.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -2428,6 +2434,7 @@ html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '
'; if (owner) { html += '
'; @@ -2468,6 +2475,7 @@ html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'wisprflow') html += '

Leave the credential source as None: the connector reads Wispr Flow\'s local history read-only with the sqlite3 CLI. Leave the path empty for the default app location on the hive host (macOS: ~/Library/Application Support/Wispr Flow/flow.sqlite; Windows: %APPDATA%\\Wispr Flow\\flow.sqlite), or point it at a copy of flow.sqlite or a folder of exported .json/.ndjson/.csv/.md files. New dictations need no connector at all — dictate into + New Fact directly.

'; if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; @@ -2632,3 +2640,11 @@ } } function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + + // ── Wispr Flow (local dictation history, no sign-in) ── + function kbWisprFlowSourceLine(c) { + const path = (c.scope && c.scope.path) || ''; + const group = (c.scope && c.scope.group) === 'day' ? 'one fact per day' : 'one fact per dictation'; + return '
Wispr Flow: ' + (path ? '' + escapeHtml(path) + '' : 'default app location on the hive host') + ' · ' + group + '
'; + } + function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 2917b2797f..07b1ff15dc 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -45497,6 +45497,12 @@

Configure in Integrations'; html += '

'; + html += '
🗣️ Wispr Flow dictation'; + html += '
'; + html += '
Dictate straight into any field here — put the cursor in + New Fact and hold the Flow key; no connector needed. To also pull your past dictations in, the Wispr Flow connector reads the app\'s local history (flow.sqlite or an export folder). $0 beyond your Wispr Flow plan.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -45944,6 +45950,7 @@ html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '
'; if (owner) { html += '
'; @@ -45984,6 +45991,7 @@ html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'wisprflow') html += '

Leave the credential source as None: the connector reads Wispr Flow\'s local history read-only with the sqlite3 CLI. Leave the path empty for the default app location on the hive host (macOS: ~/Library/Application Support/Wispr Flow/flow.sqlite; Windows: %APPDATA%\\Wispr Flow\\flow.sqlite), or point it at a copy of flow.sqlite or a folder of exported .json/.ndjson/.csv/.md files. New dictations need no connector at all — dictate into + New Fact directly.

'; if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; @@ -46148,6 +46156,14 @@ } } function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + + // ── Wispr Flow (local dictation history, no sign-in) ── + function kbWisprFlowSourceLine(c) { + const path = (c.scope && c.scope.path) || ''; + const group = (c.scope && c.scope.group) === 'day' ? 'one fact per day' : 'one fact per dictation'; + return '
Wispr Flow: ' + (path ? '' + escapeHtml(path) + '' : 'default app location on the hive host') + ' · ' + group + '
'; + } + function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } // ── Vault Management ── function kbOpenVaults() { @@ -54995,6 +55011,7 @@

Configure LiteLLM

{ tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Wispr Flow dictation connector', key: 'knowledge.connectors', lazy: true, text: 'wispr flow wisprflow dictation voice history flow.sqlite sqlite export path since_days max_items min_words group apps include_raw tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; @@ -56275,7 +56292,7 @@

Configure LiteLLM

} // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint', 'wisprflow']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -56286,7 +56303,8 @@

Configure LiteLLM

'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, - 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } + 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' }, + 'wisprflow': { label: 'Wispr Flow dictation', description: 'Sync your Wispr Flow dictation history (what you spoke, when, into which app) from the desktop app\'s local flow.sqlite or an export folder. Read-only, no sign-in, $0 beyond your Wispr Flow plan.' } }; const KC_TYPE_ICONS = { // Vendor marks: simple-icons (CC0 1.0) paths, 24x24 viewBox, rendered with currentColor. @@ -56301,6 +56319,7 @@

Configure LiteLLM

'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', + 'wisprflow': 'M3 10h2v4H3zm4-3h2v10H7zm4-5h2v20h-2zm4 5h2v10h-2zm4 3h2v4h-2z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -56318,7 +56337,8 @@

Configure LiteLLM

'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], - 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] + 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']], + 'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)', '/Users/me/Library/Application Support/Wispr Flow/flow.sqlite'], ['since_days', 'Days back on first sync', '30'], ['max_items', 'Max dictations per sync', '500'], ['min_words', 'Skip dictations shorter than (words)', '3'], ['group', 'Fact per dictation or day', 'dictation'], ['apps', 'Only these apps (comma-separated, optional)', 'slack, obsidian, vscode'], ['include_raw', 'Include raw ASR text', 'false'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['sqlite3', 'sqlite3 command (optional)', 'sqlite3']] }; let _kcState = { loaded: false, error: '', connectors: [], rows: [], types: [], layers: ['personal', 'project', 'org', 'community'], runtime: false, publish: null }; diff --git a/src/pkg/knowledge/connector/connector.go b/src/pkg/knowledge/connector/connector.go index 777984bb2b..953b89907a 100644 --- a/src/pkg/knowledge/connector/connector.go +++ b/src/pkg/knowledge/connector/connector.go @@ -216,7 +216,7 @@ func NewRegistry() *Registry { // DefaultRegistry returns a registry with the built-in connector types // (`git`, `document`, `github-wiki`, `repo-wiki`, `confluence`, `notion`, -// `sharepoint`, `google-drive`, `context7` and `plaud`) registered. +// `sharepoint`, `google-drive`, `context7`, `plaud` and `wisprflow`) registered. func DefaultRegistry() *Registry { r := NewRegistry() _ = r.Register(TypeGit, newGitConnector) @@ -229,6 +229,7 @@ func DefaultRegistry() *Registry { _ = r.Register(TypeRepoWiki, newRepoWikiConnector) _ = r.Register(TypeContext7, newContext7Connector) _ = r.Register(TypePlaud, newPlaudConnector) + _ = r.Register(TypeWisprFlow, newWisprConnector) return r } diff --git a/src/pkg/knowledge/connector/connector_test.go b/src/pkg/knowledge/connector/connector_test.go index 33e132d104..4e0c8714fa 100644 --- a/src/pkg/knowledge/connector/connector_test.go +++ b/src/pkg/knowledge/connector/connector_test.go @@ -95,7 +95,7 @@ func TestRegistryRegister(t *testing.T) { if got := r.Types(); !reflect.DeepEqual(got, []string{"another", "fake"}) { t.Fatalf("Types() = %v", got) } - if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint}) { + if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint, TypeWisprFlow}) { t.Fatalf("DefaultRegistry().Types() = %v", got) } } diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md new file mode 100644 index 0000000000..edf494c416 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md @@ -0,0 +1,3 @@ +# Dictated into Cursor + +Refactor the syncer so the wisprflow connector shares the plaud cursor parser. diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson new file mode 100644 index 0000000000..862be53254 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson @@ -0,0 +1,3 @@ +{"id":"n-1","text":"Remember to add spek, hive and dibs to the Wispr dictionary.","at":"2026-11-11 08:30:00.000 +00:00","app":"md.obsidian","words":11} + +{"id":"n-2","text":"Hi","at":"2026-11-11 08:31:00.000 +00:00","app":"com.apple.MobileSMS","words":1} diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv new file mode 100644 index 0000000000..956dbdc67a --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv @@ -0,0 +1,2 @@ +id,text,timestamp,app,numWords +c-1,"Ask Greg about the KubeCon ops checklist, then file it in dibs.",2026-11-11T09:00:00Z,com.tinyspeck.slackmacgap,13 diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json new file mode 100644 index 0000000000..d9732572c6 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json @@ -0,0 +1,4 @@ +{"dictations": [ + {"transcriptEntityId": "j-1", "formattedText": "Spec idea: a wisprflow knowledge connector for hive.", "asrText": "spec idea a wispr flow knowledge connector for hive", "timestamp": "2026-11-10T12:00:00Z", "app": "com.electron.wispr-flow", "numWords": 8}, + {"transcriptEntityId": "j-old", "formattedText": "Too old to matter.", "timestamp": "2019-05-05T12:00:00Z", "app": "com.apple.Notes"} +]} diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin b/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin new file mode 100644 index 0000000000..7c0582d8dd --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin @@ -0,0 +1 @@ +ignored binary diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md b/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md new file mode 100644 index 0000000000..0e1f51d3c3 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md @@ -0,0 +1,6 @@ +--- +id: m-fm +timestamp: 2026-11-10 16:20:00 +app: com.apple.Notes +--- +Front-matter note dictated into Apple Notes about the ACMM scorecard. diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/history.json b/src/pkg/knowledge/connector/testdata/wisprflow/history.json new file mode 100644 index 0000000000..633157e859 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/history.json @@ -0,0 +1,6 @@ +[{"id":"d-1","text":"Add a retry with backoff around the upload call.","raw":"um so add a retry with backoff around the upload call","at":"2026-11-10 09:15:00.000 +00:00","app":"com.microsoft.VSCode","url":"","duration":6.5,"numWords":9,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-2","text":"Running late — start without me.","raw":"running late start without me","at":"2026-11-11 17:55:30.000 +00:00","app":"com.tinyspeck.slackmacgap","url":"","duration":2.1,"numWords":5,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-short","text":"Ok.","raw":"ok","at":"2026-11-11 18:00:00.000 +00:00","app":"com.apple.MobileSMS","url":"","duration":0.8,"numWords":1,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-raw","text":"still processing","raw":"still processing","at":"2026-11-11 18:05:00.000 +00:00","app":"com.google.Chrome","url":"","duration":3,"numWords":2,"language":"en","status":"raw_transcript","isArchived":0}, +{"id":"d-archived","text":"Archived thought about the hive dashboard.","raw":"archived thought about the hive dashboard","at":"2026-11-11 18:10:00.000 +00:00","app":"md.obsidian","url":"","duration":4,"numWords":6,"language":"en","status":"formatted","isArchived":1}, +{"id":"d-old","text":"Ancient dictation from long ago.","raw":"ancient dictation from long ago","at":"2020-01-01 10:00:00.000 +00:00","app":"com.google.Chrome","url":"https://example.com","duration":4,"numWords":5,"language":"en","status":"formatted","isArchived":0}] diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/history.sql b/src/pkg/knowledge/connector/testdata/wisprflow/history.sql new file mode 100644 index 0000000000..1973358d97 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/history.sql @@ -0,0 +1,14 @@ +-- Minimal Wispr Flow History schema (community-documented, Sequelize DATETIME strings). +CREATE TABLE History ( + transcriptEntityId VARCHAR(36) PRIMARY KEY, + asrText TEXT, formattedText TEXT, editedText TEXT, + timestamp DATETIME, audio BLOB, screenshot BLOB, + status VARCHAR(255), app VARCHAR(255), url VARCHAR(255), + duration FLOAT, numWords INTEGER, language TEXT, isArchived TINYINT(1) DEFAULT 0 +); +INSERT INTO History VALUES ('d-1','um so add a retry with backoff around the upload call','Add a retry with backoff around the upload call.',NULL,'2026-11-10 09:15:00.000 +00:00',x'00',NULL,'formatted','com.microsoft.VSCode','',6.5,9,'en',0); +INSERT INTO History VALUES ('d-2','running late start without me','Running late, start without me.','Running late — start without me.','2026-11-11 17:55:30.000 +00:00',NULL,NULL,'formatted','com.tinyspeck.slackmacgap','',2.1,5,'en',0); +INSERT INTO History VALUES ('d-short','ok','Ok.',NULL,'2026-11-11 18:00:00.000 +00:00',NULL,NULL,'formatted','com.apple.MobileSMS','',0.8,1,'en',0); +INSERT INTO History VALUES ('d-raw','still processing','',NULL,'2026-11-11 18:05:00.000 +00:00',NULL,NULL,'raw_transcript','com.google.Chrome','',3,2,'en',0); +INSERT INTO History VALUES ('d-archived','archived thought about the hive dashboard','Archived thought about the hive dashboard.',NULL,'2026-11-11 18:10:00.000 +00:00',NULL,NULL,'formatted','md.obsidian','',4,6,'en',1); +INSERT INTO History VALUES ('d-old','ancient dictation from long ago','Ancient dictation from long ago.',NULL,'2020-01-01 10:00:00.000 +00:00',NULL,NULL,'formatted','com.google.Chrome','https://example.com',4,5,'en',0); diff --git a/src/pkg/knowledge/connector/wisprflow.go b/src/pkg/knowledge/connector/wisprflow.go new file mode 100644 index 0000000000..39f902748e --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow.go @@ -0,0 +1,577 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" +) + +// TypeWisprFlow is the connector type for Wispr Flow dictation history, +// read from the desktop app's local SQLite file or from an export folder. +const TypeWisprFlow = "wisprflow" + +const ( + // wisprDefaultSQLite3 is the sqlite3 CLI used to read flow.sqlite; it + // ships with macOS and the hive image, so no Go SQLite driver is needed. + wisprDefaultSQLite3 = "sqlite3" + // wisprDefaultSinceDays bounds the first sync. + wisprDefaultSinceDays = 30 + // wisprDefaultMaxItems caps dictations read per sync. + wisprDefaultMaxItems = 500 + // wisprDefaultMinWords drops one- and two-word dictations ("yes", "ok"). + wisprDefaultMinWords = 3 + // wisprRelistOverlap re-reads this far behind the cursor: Flow updates a + // row in place (raw_transcript → formatted), so a late finalisation is + // still picked up. + wisprRelistOverlap = 2 * time.Hour + // wisprGroupDictation emits one fact per dictation; wisprGroupDay one + // fact per calendar day. + wisprGroupDictation = "dictation" + wisprGroupDay = "day" + // wisprTitleWords is how many words of the text make the fact title. + wisprTitleWords = 10 + // wisprSQLTimeout bounds one sqlite3 invocation. + wisprSQLTimeout = 60 * time.Second + // wisprStatusFormatted is the terminal row status in Flow's History table. + wisprStatusFormatted = "formatted" +) + +// wisprDictation is one Wispr Flow history row, normalised from SQLite or +// an export file. +type wisprDictation struct { + ID string + Text string + RawText string + At time.Time + App string + URL string + Duration float64 + Words int + Language string +} + +// wisprConnector reads Wispr Flow's local dictation history. Scope keys: +// path (flow.sqlite, a backup copy, or an export folder), sqlite3, since_days, +// max_items, min_words, group (dictation|day), apps, include_raw, tags. +// No auth: the data is already on local disk. +type wisprConnector struct { + cfg ConnectorConfig + logger *slog.Logger + now func() time.Time + // query runs the sqlite3 CLI; tests replace it. + query func(ctx context.Context, dbPath, sql string) ([]byte, error) +} + +func newWisprConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { + c := &wisprConnector{cfg: cfg, logger: deps.logger(), now: time.Now} + bin := cfg.ScopeValue("sqlite3") + if bin == "" { + bin = wisprDefaultSQLite3 + } + c.query = wisprSQLite3Runner(bin) + return c, nil +} + +func (c *wisprConnector) Type() string { return TypeWisprFlow } + +// wisprDefaultPath is where the desktop app keeps its history on this OS. +func wisprDefaultPath() string { + switch runtime.GOOS { + case "darwin": + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, "Library", "Application Support", "Wispr Flow", "flow.sqlite") + case "windows": + if appdata := os.Getenv("APPDATA"); appdata != "" { + return filepath.Join(appdata, "Wispr Flow", "flow.sqlite") + } + } + return "" +} + +func (c *wisprConnector) path() string { + if p := c.cfg.ScopeValue("path"); p != "" { + return p + } + return wisprDefaultPath() +} + +// Validate checks the scope. The path may not exist yet (the app may be +// installed later); only its shape is checked. +func (c *wisprConnector) Validate(cfg ConnectorConfig) error { + if cfg.Auth.Configured() { + return fmt.Errorf("wisprflow: auth.env/auth.file are not used; the connector reads Wispr Flow's local history file") + } + p := cfg.ScopeValue("path") + if p == "" && wisprDefaultPath() == "" { + return fmt.Errorf("wisprflow: scope.path is required on %s (no default Wispr Flow location)", runtime.GOOS) + } + if p != "" && !filepath.IsAbs(p) { + return fmt.Errorf("wisprflow: scope.path must be an absolute path to flow.sqlite or an export folder") + } + for _, key := range []string{"since_days", "max_items", "min_words"} { + if v := cfg.ScopeValue(key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 || (key != "min_words" && n == 0) { + return fmt.Errorf("wisprflow: scope.%s must be a positive integer", key) + } + } + } + if g := cfg.ScopeValue("group"); g != "" && g != wisprGroupDictation && g != wisprGroupDay { + return fmt.Errorf("wisprflow: scope.group must be %q or %q", wisprGroupDictation, wisprGroupDay) + } + if v := cfg.ScopeValue("include_raw"); v != "" { + if _, err := strconv.ParseBool(v); err != nil { + return fmt.Errorf("wisprflow: scope.include_raw must be true or false") + } + } + if bin := cfg.ScopeValue("sqlite3"); bin != "" { + if _, err := splitCommandLine(bin); err != nil { + return err + } + } + return nil +} + +func (c *wisprConnector) scopeInt(key string, def int) int { + v := c.cfg.ScopeValue(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return def + } + return n +} + +func (c *wisprConnector) group() string { + if c.cfg.ScopeValue("group") == wisprGroupDay { + return wisprGroupDay + } + return wisprGroupDictation +} + +// Sync reads dictations newer than the cursor (minus an overlap) and emits +// one page per dictation or per day. The listing is incremental, so unlisted +// pages are never tombstoned. +func (c *wisprConnector) Sync(ctx context.Context, cur Cursor, emit func(Page) error) (Cursor, error) { + p := c.path() + if p == "" { + return cur, fmt.Errorf("wisprflow: no path configured") + } + info, err := os.Stat(p) + if err != nil { + return cur, fmt.Errorf("wisprflow: %w (install Wispr Flow, point scope.path at flow.sqlite or an export folder, or grant the hive process Full Disk Access on macOS)", err) + } + since := parsePlaudCursor(cur) + if since.IsZero() { + since = c.now().Add(-time.Duration(c.scopeInt("since_days", wisprDefaultSinceDays)) * 24 * time.Hour) + } else { + since = since.Add(-wisprRelistOverlap) + } + if c.group() == wisprGroupDay { + // A day page is rebuilt whole, so re-read from the start of that day. + y, m, d := since.UTC().Date() + since = time.Date(y, m, d, 0, 0, 0, 0, time.UTC) + } + maxItems := c.scopeInt("max_items", wisprDefaultMaxItems) + + var rows []wisprDictation + var partial error + if info.IsDir() { + rows, partial = readWisprExportDir(p, since, maxItems) + } else { + rows, err = c.readSQLite(ctx, p, since, maxItems) + if err != nil { + return cur, err + } + } + rows = c.filter(rows) + sort.Slice(rows, func(i, j int) bool { return rows[i].At.Before(rows[j].At) }) + + newest := parsePlaudCursor(cur) + for _, r := range rows { + if r.At.After(newest) { + newest = r.At + } + } + pages := c.pages(rows) + for _, pg := range pages { + if err := ctx.Err(); err != nil { + return cur, err + } + if err := emit(pg); err != nil { + return cur, err + } + } + next := cur + if !newest.IsZero() { + next = Cursor(newest.UTC().Format(time.RFC3339)) + } + if partial != nil { + return next, &PartialError{Err: partial} + } + return next, nil +} + +// filter applies min_words and the apps allow-list. +func (c *wisprConnector) filter(rows []wisprDictation) []wisprDictation { + minWords := c.scopeInt("min_words", wisprDefaultMinWords) + var apps []string + for _, a := range strings.Split(c.cfg.ScopeValue("apps"), ",") { + if a = strings.ToLower(strings.TrimSpace(a)); a != "" { + apps = append(apps, a) + } + } + out := rows[:0] + for _, r := range rows { + r.Text = strings.TrimSpace(r.Text) + if r.Text == "" || r.ID == "" { + continue + } + words := r.Words + if words <= 0 { + words = len(strings.Fields(r.Text)) + } + if words < minWords { + continue + } + if len(apps) > 0 && !wisprAppMatches(r.App, apps) { + continue + } + out = append(out, r) + } + return out +} + +func wisprAppMatches(app string, allow []string) bool { + app = strings.ToLower(app) + for _, a := range allow { + if strings.Contains(app, a) { + return true + } + } + return false +} + +// pages renders one page per dictation, or one per UTC day. +func (c *wisprConnector) pages(rows []wisprDictation) []Page { + includeRaw, _ := strconv.ParseBool(c.cfg.ScopeValue("include_raw")) + tags := c.cfg.ScopeValue("tags") + if c.group() == wisprGroupDictation { + out := make([]Page, 0, len(rows)) + for _, r := range rows { + out = append(out, c.dictationPage(r, includeRaw, tags)) + } + return out + } + byDay := map[string][]wisprDictation{} + var days []string + for _, r := range rows { + day := r.At.UTC().Format("2006-01-02") + if _, ok := byDay[day]; !ok { + days = append(days, day) + } + byDay[day] = append(byDay[day], r) + } + sort.Strings(days) + out := make([]Page, 0, len(days)) + for _, day := range days { + out = append(out, c.dayPage(day, byDay[day], includeRaw, tags)) + } + return out +} + +func wisprTitle(text string) string { + words := strings.Fields(text) + if len(words) > wisprTitleWords { + return strings.Join(words[:wisprTitleWords], " ") + "…" + } + return strings.Join(words, " ") +} + +// wisprAppName turns a bundle id like com.tinyspeck.slackmacgap into a +// readable label; unknown ids keep their last segment. +func wisprAppName(app string) string { + app = strings.TrimSpace(app) + if app == "" { + return "" + } + known := map[string]string{ + "com.tinyspeck.slackmacgap": "Slack", + "md.obsidian": "Obsidian", + "com.apple.MobileSMS": "Messages", + "com.microsoft.VSCode": "VS Code", + "com.google.Chrome": "Chrome", + "com.apple.Safari": "Safari", + "com.electron.wispr-flow": "Wispr Flow", + "com.openai.chat": "ChatGPT", + "com.anthropic.claudefordesktop": "Claude", + "com.apple.Notes": "Notes", + "com.apple.mail": "Mail", + "com.apple.Terminal": "Terminal", + "com.googlecode.iterm2": "iTerm2", + "com.todesktop.230313mzl4w4u92": "Cursor", + } + if n, ok := known[app]; ok { + return n + } + if i := strings.LastIndex(app, "."); i >= 0 && i < len(app)-1 { + return app[i+1:] + } + return app +} + +func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags string) Page { + appName := wisprAppName(r.App) + title := wisprTitle(r.Text) + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + fmt.Fprintf(&md, "- **Dictated:** %s\n", r.At.UTC().Format(time.RFC3339)) + if appName != "" { + fmt.Fprintf(&md, "- **App:** %s\n", appName) + } + if r.URL != "" { + fmt.Fprintf(&md, "- **URL:** %s\n", r.URL) + } + if r.Duration > 0 { + fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(r.Duration * float64(time.Second))).Round(time.Second)) + } + md.WriteString("\n") + md.WriteString(r.Text) + md.WriteString("\n") + if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { + md.WriteString("\n## Raw transcript\n\n") + md.WriteString(strings.TrimSpace(r.RawText)) + md.WriteString("\n") + } + attrs := map[string]string{ + "wispr_transcript_id": r.ID, + "dictated_at": r.At.UTC().Format(time.RFC3339), + } + if r.App != "" { + attrs["app"] = r.App + } + if r.Words > 0 { + attrs["num_words"] = strconv.Itoa(r.Words) + } + if r.Language != "" { + attrs["language"] = r.Language + } + if tags != "" { + attrs["tags"] = tags + } + return Page{ + ID: r.ID, + Title: title, + URL: r.URL, + Markdown: md.String(), + UpdatedAt: r.At, + Path: []string{"Wispr Flow", r.At.UTC().Format("2006-01-02")}, + Attrs: attrs, + } +} + +func (c *wisprConnector) dayPage(day string, rows []wisprDictation, includeRaw bool, tags string) Page { + title := "Wispr Flow dictations " + day + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + fmt.Fprintf(&md, "%d dictation(s).\n\n", len(rows)) + var newest time.Time + words := 0 + for _, r := range rows { + if r.At.After(newest) { + newest = r.At + } + words += r.Words + label := r.At.UTC().Format("15:04") + if n := wisprAppName(r.App); n != "" { + label += " · " + n + } + fmt.Fprintf(&md, "## %s\n\n%s\n\n", label, r.Text) + if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { + fmt.Fprintf(&md, "
Raw transcript\n\n%s\n\n
\n\n", strings.TrimSpace(r.RawText)) + } + } + attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows))} + if words > 0 { + attrs["num_words"] = strconv.Itoa(words) + } + if tags != "" { + attrs["tags"] = tags + } + return Page{ + ID: "day-" + day, + Title: title, + Markdown: md.String(), + UpdatedAt: newest, + Path: []string{"Wispr Flow"}, + Attrs: attrs, + } +} + +// wisprSQLite3Runner executes the sqlite3 CLI read-only with JSON output. +// The History table holds audio and screenshot BLOBs, so only text columns +// are ever selected. +func wisprSQLite3Runner(bin string) func(ctx context.Context, dbPath, sql string) ([]byte, error) { + return func(ctx context.Context, dbPath, sql string) ([]byte, error) { + argv, err := splitCommandLine(bin) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, wisprSQLTimeout) + defer cancel() + args := append(argv[1:], "-readonly", "-json", "-bail", dbPath, sql) + cmd := exec.CommandContext(ctx, argv[0], args...) + out, err := cmd.Output() + if err != nil { + var stderr string + if ee, ok := err.(*exec.ExitError); ok { + stderr = firstLine(string(ee.Stderr)) + } + if stderr == "" { + stderr = err.Error() + } + return nil, fmt.Errorf("wisprflow: sqlite3 failed: %s", stderr) + } + return out, nil + } +} + +// wisprHistorySQL selects finalised, unarchived dictations newer than since. +// Flow stores Sequelize DATETIME strings; datetime() normalises them so the +// comparison does not depend on the stored format. +func wisprHistorySQL(since time.Time, limit int) string { + s := strings.ReplaceAll(since.UTC().Format("2006-01-02 15:04:05"), "'", "''") + return fmt.Sprintf(`SELECT transcriptEntityId AS id, `+ + `COALESCE(NULLIF(editedText,''), NULLIF(formattedText,''), asrText) AS text, `+ + `asrText AS raw, timestamp AS at, COALESCE(app,'') AS app, COALESCE(url,'') AS url, `+ + `COALESCE(duration,0) AS duration, COALESCE(numWords,0) AS numWords, COALESCE(language,'') AS language `+ + `FROM History WHERE COALESCE(isArchived,0)=0 AND status='%s' AND datetime(timestamp) > datetime('%s') `+ + `ORDER BY datetime(timestamp) ASC LIMIT %d;`, wisprStatusFormatted, s, limit) +} + +func (c *wisprConnector) readSQLite(ctx context.Context, dbPath string, since time.Time, limit int) ([]wisprDictation, error) { + out, err := c.query(ctx, dbPath, wisprHistorySQL(since, limit)) + if err != nil { + return nil, err + } + return decodeWisprJSON(out) +} + +// decodeWisprJSON accepts the sqlite3 -json array (or any JSON array/object +// list of dictations, as written by community exporters). +func decodeWisprJSON(data []byte) ([]wisprDictation, error) { + text := strings.TrimSpace(string(data)) + if text == "" { + return nil, nil + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return nil, fmt.Errorf("wisprflow: invalid JSON: %s", firstLine(text)) + } + var items []any + switch v := raw.(type) { + case []any: + items = v + case map[string]any: + for _, key := range []string{"dictations", "history", "items", "data", "results", "transcripts"} { + if arr, ok := v[key].([]any); ok { + items = arr + break + } + } + if items == nil { + items = []any{v} + } + } + out := make([]wisprDictation, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]any) + if !ok { + continue + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, nil +} + +// wisprFromMap normalises one dictation object from SQLite JSON or an +// exporter's JSON, which name the same fields differently. +func wisprFromMap(m map[string]any) (wisprDictation, bool) { + d := wisprDictation{ + ID: plaudString(m, "id", "transcriptEntityId", "transcript_id", "uuid"), + Text: plaudString(m, "text", "editedText", "edited_text", "formattedText", "formatted_text", "content", "asrText"), + RawText: plaudString(m, "raw", "asrText", "asr_text", "raw_text"), + At: plaudTime(m, "at", "timestamp", "created_at", "createdAt", "date"), + App: plaudString(m, "app", "application", "bundle_id"), + URL: plaudString(m, "url"), + Language: plaudString(m, "language", "lang"), + } + d.Duration = wisprFloat(m, "duration", "duration_s", "seconds") + d.Words = int(wisprFloat(m, "numWords", "num_words", "words")) + if d.At.IsZero() { + d.At = wisprParseTime(plaudString(m, "at", "timestamp", "created_at", "createdAt", "date")) + } + if d.ID == "" || strings.TrimSpace(d.Text) == "" || d.At.IsZero() { + return d, false + } + return d, true +} + +// wisprParseTime handles Sequelize's "2026-11-11 14:05:09.123 +00:00" and +// the common ISO shapes; plaudTime covers RFC3339 and epoch numbers. +func wisprParseTime(s string) time.Time { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{} + } + for _, layout := range []string{ + "2006-01-02 15:04:05.000 -07:00", + "2006-01-02 15:04:05 -07:00", + "2006-01-02 15:04:05.000Z07:00", + "2006-01-02 15:04:05.000", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05.000Z07:00", + time.RFC3339Nano, + time.RFC3339, + "2006-01-02T15:04", + "2006-01-02 15:04", + "2006-01-02", + } { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +// wisprFloat reads the first numeric value among keys, accepting JSON +// numbers and numeric strings (CSV exports). +func wisprFloat(m map[string]any, keys ...string) float64 { + for _, k := range keys { + switch v := m[k].(type) { + case float64: + return v + case string: + if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil { + return f + } + } + } + return 0 +} diff --git a/src/pkg/knowledge/connector/wisprflow_export.go b/src/pkg/knowledge/connector/wisprflow_export.go new file mode 100644 index 0000000000..3a6d592b15 --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow_export.go @@ -0,0 +1,259 @@ +package connector + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// wisprExportMaxFileBytes caps one export file read. +const wisprExportMaxFileBytes = 64 << 20 + +// wisprFilenameTimeRe pulls a date (and optional time) from an exported +// markdown filename such as 2026-11-11-1405-slack.md or 2026-11-11T14-05-09.md. +var wisprFilenameTimeRe = regexp.MustCompile(`(\d{4}-\d{2}-\d{2})(?:[T _-](\d{2})[-:h]?(\d{2})(?:[-:m]?(\d{2}))?)?`) + +// readWisprExportDir walks an export folder (as written by `wispr export`, +// wispr-flow-exporter or a hand-made dump) and returns the dictations newer +// than since. Supported files: .json (array or wrapped object), .ndjson / +// .jsonl (one object per line), .csv (header row) and .md (one dictation per +// file, optional YAML front matter). Unreadable files are reported together +// as one error while the readable ones are still returned. +func readWisprExportDir(dir string, since time.Time, limit int) ([]wisprDictation, error) { + var out []wisprDictation + var problems []string + seen := map[string]bool{} + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + problems = append(problems, fmt.Sprintf("%s: %v", path, walkErr)) + return nil + } + name := d.Name() + if d.IsDir() { + if path != dir && strings.HasPrefix(name, ".") { + return filepath.SkipDir + } + return nil + } + if strings.HasPrefix(name, ".") { + return nil + } + rows, err := readWisprExportFile(dir, path) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v", filepath.Base(path), err)) + return nil + } + for _, r := range rows { + if r.At.IsZero() || !r.At.After(since) || seen[r.ID] { + continue + } + seen[r.ID] = true + out = append(out, r) + if len(out) >= limit { + return fs.SkipAll + } + } + return nil + }) + if err != nil { + return out, err + } + if len(problems) > 0 { + return out, fmt.Errorf("wisprflow: %d export file(s) skipped: %s", len(problems), strings.Join(problems, "; ")) + } + return out, nil +} + +// readWisprExportFile dispatches on extension; unknown extensions are ignored. +func readWisprExportFile(root, path string) ([]wisprDictation, error) { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".json", ".ndjson", ".jsonl", ".csv", ".md", ".markdown", ".txt": + default: + return nil, nil + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.Size() > wisprExportMaxFileBytes { + return nil, fmt.Errorf("file larger than %d bytes", wisprExportMaxFileBytes) + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + switch ext { + case ".json": + return decodeWisprJSON(data) + case ".ndjson", ".jsonl": + return decodeWisprNDJSON(data) + case ".csv": + return decodeWisprCSV(data) + default: + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + rel = filepath.Base(path) + } + d, ok := decodeWisprMarkdown(rel, data, info.ModTime()) + if !ok { + return nil, nil + } + return []wisprDictation{d}, nil + } +} + +func decodeWisprNDJSON(data []byte) ([]wisprDictation, error) { + var out []wisprDictation + sc := bufio.NewScanner(strings.NewReader(string(data))) + sc.Buffer(make([]byte, 0, 64<<10), wisprExportMaxFileBytes) + line := 0 + for sc.Scan() { + line++ + text := strings.TrimSpace(sc.Text()) + if text == "" { + continue + } + var m map[string]any + if err := json.Unmarshal([]byte(text), &m); err != nil { + return out, fmt.Errorf("line %d: %v", line, err) + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, sc.Err() +} + +// decodeWisprCSV maps header names onto the same keys wisprFromMap accepts. +func decodeWisprCSV(data []byte) ([]wisprDictation, error) { + r := csv.NewReader(strings.NewReader(string(data))) + r.FieldsPerRecord = -1 + r.LazyQuotes = true + header, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + return nil, nil + } + return nil, err + } + for i := range header { + header[i] = strings.TrimSpace(strings.TrimPrefix(header[i], "\ufeff")) + } + var out []wisprDictation + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return out, err + } + m := make(map[string]any, len(header)) + for i, h := range header { + if i < len(rec) && h != "" { + m[h] = rec[i] + } + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, nil +} + +// decodeWisprMarkdown reads one dictation per markdown file. Front matter +// keys id, timestamp/date/created, app and url are honoured; otherwise the +// id is the relative path and the time comes from the filename or mtime. +func decodeWisprMarkdown(rel string, data []byte, modTime time.Time) (wisprDictation, bool) { + body := strings.ReplaceAll(string(data), "\r\n", "\n") + meta := map[string]string{} + if strings.HasPrefix(body, "---\n") { + if end := strings.Index(body[4:], "\n---"); end >= 0 { + for _, line := range strings.Split(body[4:4+end], "\n") { + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + meta[strings.ToLower(strings.TrimSpace(k))] = strings.Trim(strings.TrimSpace(v), `"'`) + } + body = body[4+end+4:] + } + } + body = strings.TrimSpace(body) + // Drop a leading heading: the title is derived from the text. + if strings.HasPrefix(body, "# ") { + if i := strings.IndexByte(body, '\n'); i >= 0 { + body = strings.TrimSpace(body[i+1:]) + } else { + body = "" + } + } + d := wisprDictation{ + Text: body, + App: meta["app"], + URL: meta["url"], + Language: meta["language"], + } + d.ID = meta["id"] + if d.ID == "" { + d.ID = meta["transcriptentityid"] + } + if d.ID == "" { + d.ID = strings.TrimSuffix(filepath.ToSlash(rel), filepath.Ext(rel)) + } + for _, k := range []string{"timestamp", "dictated", "date", "created", "created_at", "createdat"} { + if v := meta[k]; v != "" { + d.At = wisprParseTime(v) + if d.At.IsZero() { + if t, ok := wisprTimeFromName(v); ok { + d.At = t + } + } + break + } + } + if d.At.IsZero() { + if t, ok := wisprTimeFromName(filepath.Base(rel)); ok { + d.At = t + } else { + d.At = modTime.UTC() + } + } + if d.Text == "" { + return d, false + } + return d, true +} + +// wisprTimeFromName extracts a UTC time from a filename-style date stamp. +func wisprTimeFromName(name string) (time.Time, bool) { + m := wisprFilenameTimeRe.FindStringSubmatch(name) + if m == nil { + return time.Time{}, false + } + stamp := m[1] + layout := "2006-01-02" + if m[2] != "" { + stamp += " " + m[2] + ":" + m[3] + layout = "2006-01-02 15:04" + if m[4] != "" { + stamp += ":" + m[4] + layout = "2006-01-02 15:04:05" + } + } + t, err := time.Parse(layout, stamp) + if err != nil { + return time.Time{}, false + } + return t.UTC(), true +} diff --git a/src/pkg/knowledge/connector/wisprflow_test.go b/src/pkg/knowledge/connector/wisprflow_test.go new file mode 100644 index 0000000000..c20df12fda --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow_test.go @@ -0,0 +1,350 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +func wisprTestNow() time.Time { return time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) } + +func newTestWispr(t *testing.T, scope map[string]string) *wisprConnector { + t.Helper() + cfg := ConnectorConfig{Name: "wispr", Type: TypeWisprFlow, Enabled: true, Layer: "personal", Scope: scope} + c, err := DefaultRegistry().New(cfg, Deps{StateDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + wc := c.(*wisprConnector) + wc.now = wisprTestNow + return wc +} + +// wisprFixtureDB builds flow.sqlite from testdata/wisprflow/history.sql with +// the sqlite3 CLI the connector itself shells out to; callers skip when the +// CLI is absent (CI runners), so the sync tests use wisprFakeQuery instead. +func wisprFixtureDB(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 CLI not installed") + } + sql, err := os.ReadFile(filepath.Join("testdata", "wisprflow", "history.sql")) + if err != nil { + t.Fatal(err) + } + db := filepath.Join(t.TempDir(), "flow.sqlite") + cmd := exec.Command("sqlite3", db) + cmd.Stdin = strings.NewReader(string(sql)) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("building fixture db: %v: %s", err, out) + } + return db +} + +var wisprSinceRE = regexp.MustCompile(`datetime\('([^']+)'\)`) + +// wisprFakeQuery stands in for the sqlite3 CLI: it serves the rows of +// testdata/wisprflow/history.json (the same data as history.sql) and applies +// the WHERE clause of wisprHistorySQL — status, isArchived and the since +// bound — so the connector sees exactly what sqlite3 -json would return. +func wisprFakeQuery(t *testing.T) func(ctx context.Context, dbPath, sql string) ([]byte, error) { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "wisprflow", "history.json")) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err := json.Unmarshal(data, &rows); err != nil { + t.Fatal(err) + } + return func(_ context.Context, dbPath, sql string) ([]byte, error) { + if _, err := os.Stat(dbPath); err != nil { + return nil, fmt.Errorf("wisprflow: sqlite3 failed: unable to open database") + } + m := wisprSinceRE.FindStringSubmatch(sql) + if m == nil || !strings.Contains(sql, "status='formatted'") { + return nil, fmt.Errorf("unexpected sql: %s", sql) + } + since, err := time.Parse("2006-01-02 15:04:05", m[1]) + if err != nil { + return nil, err + } + var out []map[string]any + for _, r := range rows { + at := wisprParseTime(r["at"].(string)) + if r["status"] != "formatted" || r["isArchived"] != float64(0) || !at.After(since) { + continue + } + out = append(out, r) + } + return json.Marshal(out) + } +} + +// newFakeWispr returns a connector whose sqlite3 runner is wisprFakeQuery +// and whose path points at an existing (empty) flow.sqlite file. +func newFakeWispr(t *testing.T, scope map[string]string) *wisprConnector { + t.Helper() + db := filepath.Join(t.TempDir(), "flow.sqlite") + if err := os.WriteFile(db, nil, 0o600); err != nil { + t.Fatal(err) + } + if scope == nil { + scope = map[string]string{} + } + scope["path"] = db + c := newTestWispr(t, scope) + c.query = wisprFakeQuery(t) + return c +} + +func collectWispr(t *testing.T, c *wisprConnector, cur Cursor) ([]Page, Cursor, error) { + t.Helper() + var pages []Page + next, err := c.Sync(context.Background(), cur, func(p Page) error { + pages = append(pages, p) + return nil + }) + return pages, next, err +} + +func wisprIDs(pages []Page) []string { + out := make([]string, 0, len(pages)) + for _, p := range pages { + out = append(out, p.ID) + } + return out +} + +func TestWisprValidate(t *testing.T) { + c := newTestWispr(t, map[string]string{"path": "/x/flow.sqlite"}) + cases := []struct { + name string + cfg ConnectorConfig + want string + }{ + {"auth rejected", ConnectorConfig{Auth: Auth{Env: "X"}, Scope: map[string]string{"path": "/x/flow.sqlite"}}, "not used"}, + {"relative path", ConnectorConfig{Scope: map[string]string{"path": "flow.sqlite"}}, "absolute"}, + {"bad group", ConnectorConfig{Scope: map[string]string{"path": "/x", "group": "week"}}, "scope.group"}, + {"bad since", ConnectorConfig{Scope: map[string]string{"path": "/x", "since_days": "0"}}, "since_days"}, + {"bad include_raw", ConnectorConfig{Scope: map[string]string{"path": "/x", "include_raw": "maybe"}}, "include_raw"}, + } + for _, tc := range cases { + err := c.Validate(tc.cfg) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want containing %q", tc.name, err, tc.want) + } + } + ok := ConnectorConfig{Scope: map[string]string{"path": "/x/flow.sqlite", "group": "day", "since_days": "7", "min_words": "0", "include_raw": "true"}} + if err := c.Validate(ok); err != nil { + t.Fatalf("valid config rejected: %v", err) + } +} + +func TestWisprSyncSQLite(t *testing.T) { + c := newFakeWispr(t, map[string]string{"tags": "voice", "include_raw": "true"}) + + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + // d-short (1 word), d-raw (not formatted), d-archived and d-old (before + // since_days) are all excluded. + if got := wisprIDs(pages); strings.Join(got, ",") != "d-1,d-2" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T17:55:30Z" { + t.Fatalf("cursor = %q", cur) + } + p := pages[1] + if p.Title != "Running late — start without me." { + t.Errorf("title = %q (editedText should win)", p.Title) + } + for _, want := range []string{"**App:** Slack", "**Dictated:** 2026-11-11T17:55:30Z", "## Raw transcript", "running late start without me"} { + if !strings.Contains(p.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, p.Markdown) + } + } + if p.Attrs["wispr_transcript_id"] != "d-2" || p.Attrs["app"] != "com.tinyspeck.slackmacgap" || p.Attrs["tags"] != "voice" || p.Attrs["num_words"] != "5" { + t.Errorf("attrs = %v", p.Attrs) + } + if strings.Join(p.Path, "/") != "Wispr Flow/2026-11-11" { + t.Errorf("path = %v", p.Path) + } + if _, ok := pages[0].Attrs["tags"]; !ok || strings.Contains(pages[0].Markdown, "## Raw transcript") == false { + t.Errorf("first page attrs/raw = %v", pages[0].Attrs) + } + + // Incremental: the overlap window re-reads d-2 only. + pages, cur2, err := collectWispr(t, c, cur) + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "d-2" { + t.Fatalf("incremental pages = %v", got) + } + if cur2 != cur { + t.Fatalf("cursor moved to %q", cur2) + } +} + +func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { + c := newFakeWispr(t, map[string]string{"apps": "slack, vscode", "group": "day", "since_days": "3650"}) + pages, _, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "day-2026-11-10,day-2026-11-11" { + t.Fatalf("pages = %v", got) + } + day := pages[1] + if day.Attrs["dictations"] != "1" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { + t.Errorf("day page = %v\n%s", day.Attrs, day.Markdown) + } + if strings.Contains(day.Markdown, "Ok.") { + t.Error("one-word dictation leaked into the day page") + } + // d-old is Chrome, so even with a 10-year window the apps filter drops it. + for _, p := range pages { + if strings.Contains(p.Markdown, "Ancient") { + t.Error("apps filter did not drop the Chrome dictation") + } + } +} + +// TestWisprSQLite3CLI runs the real sqlite3 CLI against the fixture database +// and checks it agrees with the fake used by the other tests. +func TestWisprSQLite3CLI(t *testing.T) { + db := wisprFixtureDB(t) + c := newTestWispr(t, map[string]string{"path": db, "since_days": "3650"}) + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "d-old,d-1,d-2" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T17:55:30Z" { + t.Fatalf("cursor = %q", cur) + } +} + +func TestWisprSyncMissingPath(t *testing.T) { + c := newTestWispr(t, map[string]string{"path": filepath.Join(t.TempDir(), "nope", "flow.sqlite")}) + _, cur, err := collectWispr(t, c, "2026-01-01T00:00:00Z") + if err == nil || !strings.Contains(err.Error(), "Full Disk Access") { + t.Fatalf("err = %v", err) + } + if cur != "2026-01-01T00:00:00Z" { + t.Fatalf("cursor changed on error: %q", cur) + } +} + +func TestWisprSyncSQLiteError(t *testing.T) { + abs, _ := filepath.Abs(filepath.Join("testdata", "wisprflow", "history.sql")) + c := newTestWispr(t, map[string]string{"path": abs}) + c.query = func(context.Context, string, string) ([]byte, error) { + return nil, &PartialError{Err: os.ErrPermission} + } + if _, _, err := collectWispr(t, c, ""); err == nil { + t.Fatal("expected sqlite error to surface") + } +} + +func TestWisprHistorySQL(t *testing.T) { + sql := wisprHistorySQL(time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC), 7) + for _, want := range []string{"status='formatted'", "datetime('2026-11-01 12:00:00')", "LIMIT 7", "COALESCE(isArchived,0)=0", "NULLIF(editedText,'')"} { + if !strings.Contains(sql, want) { + t.Errorf("sql missing %q: %s", want, sql) + } + } + if strings.Contains(sql, "audio") || strings.Contains(sql, "screenshot") { + t.Error("sql must not select BLOB columns") + } +} + +func TestWisprSyncExportDir(t *testing.T) { + dir, _ := filepath.Abs(filepath.Join("testdata", "wisprflow", "export")) + c := newTestWispr(t, map[string]string{"path": dir}) + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + // j-old is before since_days; n-2 is one word; ignored.bin is skipped. + if got := wisprIDs(pages); strings.Join(got, ",") != "j-1,m-fm,n-1,c-1,2026-11-11-1430-cursor" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T14:30:00Z" { + t.Fatalf("cursor = %q", cur) + } + byID := map[string]Page{} + for _, p := range pages { + byID[p.ID] = p + } + if p := byID["2026-11-11-1430-cursor"]; !strings.HasPrefix(p.Title, "Refactor the syncer") || strings.Contains(p.Markdown, "# Dictated into Cursor") { + t.Errorf("markdown export page = %q\n%s", p.Title, p.Markdown) + } + if p := byID["m-fm"]; p.Attrs["app"] != "com.apple.Notes" || !p.UpdatedAt.Equal(time.Date(2026, 11, 10, 16, 20, 0, 0, time.UTC)) { + t.Errorf("front-matter page = %v %v", p.Attrs, p.UpdatedAt) + } + if p := byID["c-1"]; p.Attrs["num_words"] != "13" || !strings.Contains(p.Markdown, "**App:** Slack") { + t.Errorf("csv page = %v\n%s", p.Attrs, p.Markdown) + } + if p := byID["j-1"]; !strings.Contains(p.Markdown, "**App:** Wispr Flow") { + t.Errorf("json page:\n%s", p.Markdown) + } +} + +func TestWisprExportDirPartial(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "bad.json"), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "good.ndjson"), []byte(`{"id":"g","text":"three words here","at":"2026-11-11T10:00:00Z"}`+"\n"), 0o600); err != nil { + t.Fatal(err) + } + c := newTestWispr(t, map[string]string{"path": dir}) + pages, cur, err := collectWispr(t, c, "") + if !IsPartial(err) || !strings.Contains(err.Error(), "bad.json") { + t.Fatalf("err = %v, want partial naming bad.json", err) + } + if len(pages) != 1 || pages[0].ID != "g" || cur != "2026-11-11T10:00:00Z" { + t.Fatalf("pages = %v cur = %q", wisprIDs(pages), cur) + } +} + +func TestWisprParseTime(t *testing.T) { + cases := map[string]time.Time{ + "2026-11-11 17:55:30.000 +00:00": time.Date(2026, 11, 11, 17, 55, 30, 0, time.UTC), + "2026-11-11 19:55:30.250 +02:00": time.Date(2026, 11, 11, 17, 55, 30, 250e6, time.UTC), + "2026-11-11T17:55:30Z": time.Date(2026, 11, 11, 17, 55, 30, 0, time.UTC), + "2026-11-11": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + "garbage": {}, + } + for in, want := range cases { + if got := wisprParseTime(in); !got.Equal(want) { + t.Errorf("wisprParseTime(%q) = %v, want %v", in, got, want) + } + } + if got, ok := wisprTimeFromName("2026-11-11T14-05-09.md"); !ok || !got.Equal(time.Date(2026, 11, 11, 14, 5, 9, 0, time.UTC)) { + t.Errorf("wisprTimeFromName = %v %v", got, ok) + } + if _, ok := wisprTimeFromName("notes.md"); ok { + t.Error("wisprTimeFromName matched a name without a date") + } +} + +func TestWisprAppName(t *testing.T) { + for in, want := range map[string]string{"com.tinyspeck.slackmacgap": "Slack", "com.example.foo": "foo", "": "", "Cursor": "Cursor"} { + if got := wisprAppName(in); got != want { + t.Errorf("wisprAppName(%q) = %q, want %q", in, got, want) + } + } +} From efb2a0bab5b6d604ef898cae5e10783de00404a4 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:03:09 +0000 Subject: [PATCH 066/223] =?UTF-8?q?=F0=9F=90=9B=20fix(agent):=20replace=20?= =?UTF-8?q?unopenable=20pluk=20session=20log=20so=20the=20supervisor=20can?= =?UTF-8?q?=20create=20it=20(#17889)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-pluk-log-dir-permission.md | 1 + src/crux/agent/pluk_dirs.go | 12 +++++++ src/crux/agent/pluk_dirs_branches_test.go | 36 ++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 changelog.d/fixed-pluk-log-dir-permission.md diff --git a/changelog.d/fixed-pluk-log-dir-permission.md b/changelog.d/fixed-pluk-log-dir-permission.md new file mode 100644 index 0000000000..4056c1f6f9 --- /dev/null +++ b/changelog.d/fixed-pluk-log-dir-permission.md @@ -0,0 +1 @@ +- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). diff --git a/src/crux/agent/pluk_dirs.go b/src/crux/agent/pluk_dirs.go index c005c1dc0e..a8bd56c0c2 100644 --- a/src/crux/agent/pluk_dirs.go +++ b/src/crux/agent/pluk_dirs.go @@ -1,7 +1,9 @@ package agent import ( + "errors" "fmt" + "io/fs" "os" "path/filepath" ) @@ -50,6 +52,16 @@ func ensurePlukLogFile(runDir, session string) (string, error) { path := plukSessionLogPath(runDir, session) f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + if errors.Is(err, fs.ErrPermission) { + // A log left behind by another uid (a pane shell's `>>` under a tight + // umask, or an earlier run as a different user) cannot be opened or + // chmodded by us, but the shared setgid logs dir is group-writable, so + // the stale file can be replaced. Replacing beats leaving the session's + // log unreadable to peers. + if rmErr := os.Remove(path); rmErr == nil { + f, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + } + } if err != nil { return "", fmt.Errorf("creating %s: %w", path, err) } diff --git a/src/crux/agent/pluk_dirs_branches_test.go b/src/crux/agent/pluk_dirs_branches_test.go index 45e6c39179..5cac61004b 100644 --- a/src/crux/agent/pluk_dirs_branches_test.go +++ b/src/crux/agent/pluk_dirs_branches_test.go @@ -83,6 +83,42 @@ func TestEnsurePlukLogFileWidensExistingTightLog(t *testing.T) { } } +// TestEnsurePlukLogFileReplacesUnopenableLog pins the supervisor failure: a log +// left by another uid that we cannot open (modelled as mode 0000) is replaced +// in the group-writable logs dir instead of failing with permission denied. +func TestEnsurePlukLogFileReplacesUnopenableLog(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses file permission checks") + } + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + + path := plukSessionLogPath(runDir, "hive-supervisor") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("seed log: %v", err) + } + if err := os.Chmod(path, 0); err != nil { + t.Fatalf("lock log: %v", err) + } + + got, err := ensurePlukLogFile(runDir, "hive-supervisor") + if err != nil { + t.Fatalf("ensure pluk log file with unopenable log: %v", err) + } + if got != path { + t.Errorf("returned path = %q, want %q", got, path) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if perm := info.Mode().Perm(); perm != 0o660 { + t.Errorf("replaced log mode = %04o, want 0660", perm) + } +} + // TestEnsurePlukLogFileErrorWhenLogsDirMissing covers the OpenFile error // return: with no logs/ directory the create fails, the error is wrapped // (fs.ErrNotExist stays reachable through errors.Is), and it names the path. From d7ada375639f6f444e9885776fc66139085b1db4 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 07:05:44 +0000 Subject: [PATCH 067/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.6 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-17696-app-repo-coverage.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-17696-app-repo-coverage.md diff --git a/CHANGELOG.md b/CHANGELOG.md index da7895396c..a5948e1c87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.6) + +### Fixed + +- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). + ## 2026-11-12 (v5.228.5) ### Fixed diff --git a/changelog.d/fixed-17696-app-repo-coverage.md b/changelog.d/fixed-17696-app-repo-coverage.md deleted file mode 100644 index 8855f73672..0000000000 --- a/changelog.d/fixed-17696-app-repo-coverage.md +++ /dev/null @@ -1 +0,0 @@ -- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). From ccb0bf1f262d3de6b21d36122f96408c2f2f68ad Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Thu, 12 Nov 2026 00:10:12 -0700 Subject: [PATCH 068/223] =?UTF-8?q?=E2=9C=A8=20workflows:=20owner=20toggle?= =?UTF-8?q?=20for=20model-based=20stage=20derivation=20from=20the=20dashbo?= =?UTF-8?q?ard=20(#17886)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ workflows: owner toggle for model-based stage derivation from the dashboard workflow.enabled gates rules-based stage derivation and bound-model label provisioning, but could only be set by editing hive.yaml, which hosted hives cannot do. Add PUT /api/workflows/enabled {enabled} (owner-gated, persists hive.yaml, audited workflow_enabled / workflow_disabled) and a "Derive stages from bound models" switch in Settings → Workflows → Library that confirms before turning on. The stage engine and the tracker already read cfg.Workflow.Enabled live and the reloader mutates the same *Config in place, so no reload hook is needed. Turning on also provisions every bound repository's model labels at once through the sweep's memoised ensureStageLabels path (new automerge.Engine.EnsureStageLabels, wired as Dependencies.WorkflowProvisionLabels); failures come back as warnings and the next sweep retries. The Repos binding row says "columns only — turn on derivation in Workflows" when a binding exists while the flag is off. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson * chore: move changelog fragment to root changelog.d/ Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/added-workflow-enabled-toggle.md | 1 + dashboard/openapi.json | 72 +++++++++++++ src/cmd/hive/main.go | 11 +- src/cmd/hive/workflow_labels.go | 42 ++++++++ src/cmd/hive/workflow_labels_test.go | 28 +++++ src/docs/api-reference.md | 2 +- src/docs/api-reference/configuration.md | 1 + src/docs/design/workflow-models.md | 16 +++ src/hive.yaml.example | 11 ++ src/pkg/dashboard/api_workflows.go | 87 +++++++++++++++ src/pkg/dashboard/api_workflows_test.go | 102 ++++++++++++++++++ src/pkg/dashboard/deps.go | 4 + .../dashboard/static-src/css/workflows.css | 4 + .../static-src/js/config-workflows-library.js | 67 +++++++++++- .../static-src/js/config-workflows.js | 4 +- src/pkg/dashboard/static/index.html | 75 +++++++++++-- .../dashboard/workflows_library_ui_test.go | 56 ++++++++++ .../automerge/automerge_stage_labels_gc.go | 15 +++ .../automerge_stage_labels_workflow_test.go | 30 ++++++ 19 files changed, 610 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-workflow-enabled-toggle.md create mode 100644 src/cmd/hive/workflow_labels.go create mode 100644 src/cmd/hive/workflow_labels_test.go diff --git a/changelog.d/added-workflow-enabled-toggle.md b/changelog.d/added-workflow-enabled-toggle.md new file mode 100644 index 0000000000..44fc411700 --- /dev/null +++ b/changelog.d/added-workflow-enabled-toggle.md @@ -0,0 +1 @@ +- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index cc87eca843..8ecb48243e 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -36029,6 +36029,78 @@ } } }, + "/api/workflows/enabled": { + "put": { + "tags": [ + "Workflows" + ], + "summary": "Turn model-based stage derivation on or off", + "description": "Owner only. Sets workflow.enabled from the dashboard so a hosted hive with no hive.yaml access can enable it (hivecommons/hive#17620). Body {enabled}. Persists hive.yaml; the stage engine and the tracker read the live flag, so it takes effect on the next derivation pass without a restart. Turning it on also provisions every bound repository's model labels immediately; a repository that could not be provisioned is listed in warnings and the next sweep retries it. Audited as workflow_enabled / workflow_disabled; a no-op is not audited.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Flag after the change", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "changed": { + "type": "boolean" + }, + "bindings": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowBinding" + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Bound repositories whose labels could not be provisioned now (only when turning on)" + } + } + } + } + } + }, + "400": { + "description": "Body is not {enabled: boolean}" + }, + "403": { + "description": "Not an owner" + }, + "500": { + "description": "hive.yaml could not be saved; the flag is left unchanged" + }, + "503": { + "description": "No config" + } + } + } + }, "/api/workflows/{name}/{version}": { "get": { "tags": [ diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 51f69d747e..9062bf39e5 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1094,11 +1094,12 @@ func (b *boot) wireBootClosures() { WorkflowItemSnapshot: func(ctx context.Context, repo string, number int) (github.WorkflowItemSnapshot, error) { return b.ghClient.WorkflowItemSnapshot(ctx, repo, number) }, - WorkflowRepoFile: workflowRepoFile(func() *github.Client { return b.ghClient }), - WorkflowClusterRunner: workflowClusterRunner(), - WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), - AgentMgr: b.agentMgr, - CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), + WorkflowRepoFile: workflowRepoFile(func() *github.Client { return b.ghClient }), + WorkflowProvisionLabels: workflowProvisionLabels(b.cfg, func() *github.Client { return b.ghClient }, b.logger), + WorkflowClusterRunner: workflowClusterRunner(), + WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), + AgentMgr: b.agentMgr, + CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ // linearagent adapters behind the dashboard's consumer-defined // interfaces — this composition root is the only non-test place that diff --git a/src/cmd/hive/workflow_labels.go b/src/cmd/hive/workflow_labels.go new file mode 100644 index 0000000000..df5f076da3 --- /dev/null +++ b/src/cmd/hive/workflow_labels.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" +) + +// workflowProvisionLabels is the dashboard's WorkflowProvisionLabels +// dependency (hivecommons/hive#17620): when an owner turns workflow.enabled +// on from Settings → Workflows, every bound repository's model labels are +// provisioned at once through the sweep's own memoised path +// (automerge.Engine.EnsureStageLabels over the process-shared stage engine +// options and tracker), rather than on the next PR that needs a label. +// Without a GitHub client the call reports an error and the handler +// surfaces it as a warning; the flag itself still flips. +func workflowProvisionLabels(cfg *config.Config, client func() *github.Client, logger *slog.Logger) func(ctx context.Context, repo string) error { + return func(ctx context.Context, repo string) error { + var gh *github.Client + if client != nil { + gh = client() + } + if gh == nil { + return errNoForgeClient + } + owner, name, ok := strings.Cut(config.QualifyRepo(cfg.Project.Org, repo), "/") + if !ok || owner == "" || name == "" { + return fmt.Errorf("repository %q is not owner/name", repo) + } + eng := automerge.New(gh, automerge.Options{ + Logger: logger, + StageLabelPolicy: stageLabelPolicyFunc(cfg), + StageEngine: stageEngineOptions(cfg, logger), + }) + return eng.EnsureStageLabels(ctx, owner, name) + } +} diff --git a/src/cmd/hive/workflow_labels_test.go b/src/cmd/hive/workflow_labels_test.go new file mode 100644 index 0000000000..205caf29a8 --- /dev/null +++ b/src/cmd/hive/workflow_labels_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +func TestWorkflowProvisionLabelsWithoutClient(t *testing.T) { + cfg := &config.Config{Project: config.ProjectConfig{Org: "acme"}} + provision := workflowProvisionLabels(cfg, func() *github.Client { return nil }, nil) + if err := provision(context.Background(), "widgets"); !errors.Is(err, errNoForgeClient) { + t.Fatalf("err = %v", err) + } +} + +func TestWorkflowProvisionLabelsRejectsBareRepoWithoutOrg(t *testing.T) { + cfg := &config.Config{} + provision := workflowProvisionLabels(cfg, func() *github.Client { return &github.Client{} }, nil) + err := provision(context.Background(), "widgets") + if err == nil || !strings.Contains(err.Error(), "owner/name") { + t.Fatalf("err = %v", err) + } +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 44a4feadf9..33ea39c580 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,7 +13,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Health, status, events | [`api-reference/health-status-events.md`](api-reference/health-status-events.md) | 50 | | Snapshots and style | [`api-reference/snapshots-and-style.md`](api-reference/snapshots-and-style.md) | 5 | | Auth and external accounts | [`api-reference/auth.md`](api-reference/auth.md) | 22 | -| Configuration | [`api-reference/configuration.md`](api-reference/configuration.md) | 105 | +| Configuration | [`api-reference/configuration.md`](api-reference/configuration.md) | 110 | | Agents and controls | [`api-reference/agents-and-controls.md`](api-reference/agents-and-controls.md) | 36 | | Packs and ACMM | [`api-reference/packs-and-acmm.md`](api-reference/packs-and-acmm.md) | 18 | | Cost, tokens, telemetry | [`api-reference/cost-tokens-telemetry.md`](api-reference/cost-tokens-telemetry.md) | 13 | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 00f31ea1e5..4941f8f958 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -117,3 +117,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Set | `pkg/dashboard/api.go:282` | | `GET` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Get | `pkg/dashboard/api.go:297` | | `PUT` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Set | `pkg/dashboard/api.go:298` | +| `PUT` | `/api/workflows/enabled` | Owner only | Turn model-based stage derivation (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) on or off from the dashboard, for hosted hives with no `hive.yaml` access. Body `{enabled: bool}`; persists `hive.yaml`; the stage engine and tracker read the live flag so it applies on the next derivation pass without a restart. Turning on also provisions every bound repository's model labels now (`warnings[]` lists repositories that could not be provisioned; the next sweep retries). Returns `{enabled, changed, bindings[], warnings?[]}`. Audited `workflow_enabled` / `workflow_disabled`; `400` on a non-boolean body | `pkg/dashboard/api_workflows.go:28` | diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index 08d1e7f5cd..cba8a0dd33 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -169,6 +169,22 @@ and `GET /api/workflows/{name}/{version}`: Duplicate (saves a copy under a new name at version 1, provenance `local`) and Delete; Delete is withheld while anything is bound (the API returns 409 anyway) and the row says `bound` instead. Export shows the model JSON in-app (copy to clipboard) for everyone. +- **Derivation switch** — the Library header carries an owner-only toggle, *Derive stages from + bound models* (`renderWorkflowEnabledToggle`, `PUT /api/workflows/enabled {enabled}`), so a + hosted hive with no `hive.yaml` access can turn `workflow.enabled` on or off. Off: a binding + only changes the Activity Monitor's columns. On: a bound repository's stages are derived by its + model's rules and the model's labels are provisioned. Turning on asks for confirmation (it + changes derivation for every bound repository and creates labels in them); turning off is + immediate. The handler sets `Config.Workflow.Enabled`, saves `hive.yaml`, audits + `workflow_enabled` / `workflow_disabled`, and — on the way on — calls + `Dependencies.WorkflowProvisionLabels` for each binding (cmd/hive wires it to + `automerge.Engine.EnsureStageLabels`, the sweep's memoised `ensureStageLabels` path over the + shared stage-engine options and tracker), so the bound models' labels exist at once rather than + on the next PR that needs a label write; a repository that fails is returned in `warnings[]` + and the next sweep retries. No reload hook is needed: `stageEngineWorkflowFor` and the tracker + read `cfg.Workflow.Enabled` on every call, and the hive.yaml reloader mutates the same + `*Config` in place. The Repos binding row shows *columns only — turn on derivation in + Workflows* when a binding exists while the flag is off. - **Detail** — facts, the diagram, diagnostics, citations and the raw model/profile JSON. `workflowDiagramSvg(model, profile)` is a pure function: the compiled profile's columns are lanes in display order, each state is a chip carrying its padded stage number, transitions are bezier diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 3277db7eba..cbd09db9a6 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1290,3 +1290,14 @@ data: claude_sessions_dir: /data/home/.claude/projects copilot_sessions_dir: /data/home/.copilot/session-state agents_dir: /data/agent-configs + +# Workflow models (hivecommons/hive#17620, docs/design/workflow-models.md). +# Repositories are bound to a model on Settings → Repos (project.workflow_bindings). +# A binding always changes the Activity Monitor's columns for that repository; +# `enabled` decides whether the bound model's rules also derive its stages and +# whether the model's labels are provisioned in the repository. Owners can flip +# it from Settings → Workflows → Library (PUT /api/workflows/enabled) without +# editing this file; the change applies on the next derivation pass. +# workflow: +# enabled: false # false (default): bindings are columns only +# label_gc: true # delete labels a previous binding provisioned once nothing wears them diff --git a/src/pkg/dashboard/api_workflows.go b/src/pkg/dashboard/api_workflows.go index d937eca0b9..71769ec9b8 100644 --- a/src/pkg/dashboard/api_workflows.go +++ b/src/pkg/dashboard/api_workflows.go @@ -1,6 +1,7 @@ package dashboard import ( + "context" "errors" "fmt" "net/http" @@ -24,6 +25,7 @@ func (s *Server) registerWorkflowRoutes() { s.mux.HandleFunc("POST /api/workflows/ingest", s.handleWorkflowIngest) s.mux.HandleFunc("GET /api/workflows/bindings", s.handleWorkflowBindingsGet) s.mux.HandleFunc("PUT /api/workflows/bindings", s.handleWorkflowBindingPut) + s.mux.HandleFunc("PUT /api/workflows/enabled", s.handleWorkflowEnabledPut) s.mux.HandleFunc("GET /api/workflows/{name}/{version}", s.handleWorkflowGet) s.mux.HandleFunc("PUT /api/workflows/{name}/{version}", s.handleWorkflowSave) s.mux.HandleFunc("DELETE /api/workflows/{name}/{version}", s.handleWorkflowDelete) @@ -236,3 +238,88 @@ func (s *Server) handleWorkflowBindingPut(w http.ResponseWriter, r *http.Request } jsonResponse(w, map[string]interface{}{"changed": changed, "bindings": s.workflowBindings()}) } + +// workflowProvisionTimeout bounds the label provisioning PUT +// /api/workflows/enabled runs across every bound repository when it turns +// derivation on: one label list plus a create per missing label per +// repository, all within the request. +const workflowProvisionTimeout = 60 * time.Second + +// handleWorkflowEnabledPut flips workflow.enabled (hivecommons/hive#17620) +// from the dashboard so a hosted hive with no hive.yaml access can turn +// model-based stage derivation on or off. The stage engine +// (cmd/hive stageEngineWorkflowFor) and the tracker read the live config +// on every call, so the change takes effect on the next derivation pass +// with no restart or reload. Turning it on also provisions every bound +// repository's model labels now through Dependencies.WorkflowProvisionLabels; +// a provisioning failure is reported in the response as a warning, never as +// a failure of the flip itself, because the sweep retries it on its next +// pass. Audited as workflow_enabled / workflow_disabled; a no-op flip is +// not audited. +func (s *Server) handleWorkflowEnabledPut(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "config unavailable", http.StatusServiceUnavailable) + return + } + var body struct { + Enabled *bool `json:"enabled"` + } + if err := decodeBody(r, &body); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if body.Enabled == nil { + jsonError(w, "body must be {\"enabled\": true|false}", http.StatusBadRequest) + return + } + changed := s.deps.Config.Workflow.Enabled != *body.Enabled + if changed { + s.deps.Config.Workflow.Enabled = *body.Enabled + if err := s.saveConfig(); err != nil { + s.deps.Config.Workflow.Enabled = !*body.Enabled + jsonError(w, "save config: "+err.Error(), http.StatusInternalServerError) + return + } + action := "workflow_disabled" + if *body.Enabled { + action = "workflow_enabled" + } + s.auditFromRequest(r, action, fmt.Sprintf("%d bound repositories", len(s.workflowBindings())), "") + } + resp := map[string]interface{}{ + "enabled": s.deps.Config.Workflow.Enabled, + "changed": changed, + "bindings": s.workflowBindings(), + } + if *body.Enabled { + if warnings := s.provisionBoundWorkflowLabels(r.Context()); len(warnings) > 0 { + resp["warnings"] = warnings + } + } + jsonResponse(w, resp) +} + +// provisionBoundWorkflowLabels runs WorkflowProvisionLabels for every bound +// repository and returns one message per repository it could not provision. +func (s *Server) provisionBoundWorkflowLabels(ctx context.Context) []string { + if s.deps.WorkflowProvisionLabels == nil { + return nil + } + bindings := s.workflowBindings() + if len(bindings) == 0 { + return nil + } + ctx, cancel := context.WithTimeout(ctx, workflowProvisionTimeout) + defer cancel() + var warnings []string + for _, b := range bindings { + if err := s.deps.WorkflowProvisionLabels(ctx, b.Repo); err != nil { + s.logger.Warn("workflow labels: provisioning on enable failed; the next sweep retries", "repo", b.Repo, "workflow", b.Key(), "err", err) + warnings = append(warnings, fmt.Sprintf("%s: %s", b.Repo, err.Error())) + } + } + return warnings +} diff --git a/src/pkg/dashboard/api_workflows_test.go b/src/pkg/dashboard/api_workflows_test.go index bb9e077957..7a7a7d9c59 100644 --- a/src/pkg/dashboard/api_workflows_test.go +++ b/src/pkg/dashboard/api_workflows_test.go @@ -2,12 +2,16 @@ package dashboard import ( "bytes" + "context" "encoding/json" + "errors" "io" "log/slog" "net/http" "net/http/httptest" "os" + "path/filepath" + "strings" "testing" "time" @@ -276,3 +280,101 @@ func TestStageTrackerWorkflowsBlock(t *testing.T) { t.Fatalf("%+v", resp.Workflows) } } + +// PUT /api/workflows/enabled flips workflow.enabled from the dashboard +// (hivecommons/hive#17620): owner-gated, persisted to hive.yaml, audited, +// and on the way on it provisions every bound repository's labels. +func TestWorkflowEnabledPut(t *testing.T) { + s := workflowTestServer(t, true) + s.deps.Config.SourcePath = filepath.Join(t.TempDir(), "hive.yaml") + var provisioned []string + s.deps.WorkflowProvisionLabels = func(_ context.Context, repo string) error { + provisioned = append(provisioned, repo) + if repo == "repo2" { + return errors.New("no such repository") + } + return nil + } + if rec := wfReq(s, http.MethodPost, "/api/workflows", techdocsModel(t), true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + for _, repo := range []string{"repo1", "repo2"} { + if _, err := s.deps.Config.SetWorkflowBinding(repo, "cncf-techdocs", "1"); err != nil { + t.Fatal(err) + } + } + + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d %s", rec.Code, rec.Body) + } + for _, bad := range []string{"{", "{}", `{"enabled":"yes"}`} { + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", bad, true); rec.Code != http.StatusBadRequest { + t.Fatalf("body %s: %d %s", bad, rec.Code, rec.Body) + } + } + if s.deps.Config.Workflow.Enabled || len(provisioned) != 0 { + t.Fatalf("rejected requests changed state: enabled=%v provisioned=%v", s.deps.Config.Workflow.Enabled, provisioned) + } + + rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true) + var out struct { + Enabled bool + Changed bool + Bindings []struct{ Repo string } + Warnings []string + } + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || !out.Enabled || !out.Changed || len(out.Bindings) != 2 { + t.Fatalf("enable: %d %s", rec.Code, rec.Body) + } + if !s.deps.Config.Workflow.Enabled { + t.Fatal("flag not applied to the live config") + } + if len(provisioned) != 2 || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "repo2") { + t.Fatalf("provisioned=%v warnings=%v", provisioned, out.Warnings) + } + saved, err := os.ReadFile(s.deps.Config.SourcePath) + if err != nil || !bytes.Contains(saved, []byte("enabled: true")) { + t.Fatalf("hive.yaml not persisted: %v\n%s", err, saved) + } + rec = wfReq(s, http.MethodGet, "/api/workflows", nil, false) + if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte(`"enabled":true`)) { + t.Fatalf("list after enable: %d %s", rec.Code, rec.Body) + } + + // Idempotent: same value again is not a change, not audited, but still + // re-provisions (the memo makes that cheap). + rec = wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || !out.Enabled || out.Changed { + t.Fatalf("enable again: %d %s", rec.Code, rec.Body) + } + provisioned = nil + out.Warnings = nil + rec = wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": false}, true) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || out.Enabled || !out.Changed || len(out.Warnings) != 0 { + t.Fatalf("disable: %d %s", rec.Code, rec.Body) + } + if s.deps.Config.Workflow.Enabled || len(provisioned) != 0 { + t.Fatalf("disable: enabled=%v provisioned=%v", s.deps.Config.Workflow.Enabled, provisioned) + } + + var actions []string + for _, e := range s.audit.Recent(0) { + if e.Action == "workflow_enabled" || e.Action == "workflow_disabled" { + actions = append(actions, e.Action) + } + } + if strings.Join(actions, ",") != "workflow_disabled,workflow_enabled" && strings.Join(actions, ",") != "workflow_enabled,workflow_disabled" { + t.Fatalf("audit actions %v", actions) + } + + // A failed save leaves the live flag where it was. + s.deps.Config.SourcePath = filepath.Join(t.TempDir(), "missing", "hive.yaml") + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true); rec.Code != http.StatusInternalServerError || s.deps.Config.Workflow.Enabled { + t.Fatalf("unsaveable: %d %s enabled=%v", rec.Code, rec.Body, s.deps.Config.Workflow.Enabled) + } + + s.deps.Config = nil + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no config: %d", rec.Code) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 251c12bddb..e0bff6c556 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -54,6 +54,10 @@ type Dependencies struct { // spec) for POST /api/workflows/ingest/spec {repo, path}. Nil disables // repo-path spec ingest. WorkflowRepoFile func(ctx context.Context, repo, path string) (string, error) + // WorkflowProvisionLabels creates (or recolours) one bound repository's + // stage and model labels now, for PUT /api/workflows/enabled turning + // derivation on. Nil leaves provisioning to the next sweep pass. + WorkflowProvisionLabels func(ctx context.Context, repo string) error // WorkflowClusterRunner runs kubectl for POST /api/workflows/ingest/cluster // (RFC #17620 phase 6). Nil means no kube access is configured on this // spoke and the route answers 422. diff --git a/src/pkg/dashboard/static-src/css/workflows.css b/src/pkg/dashboard/static-src/css/workflows.css index 7ae7bc94b8..0fcbaf339f 100644 --- a/src/pkg/dashboard/static-src/css/workflows.css +++ b/src/pkg/dashboard/static-src/css/workflows.css @@ -6,6 +6,10 @@ .wf-key { color: var(--text); text-decoration: none; font-weight: 700; } .wf-key:hover { text-decoration: underline; } .wf-desc { font-size: var(--fs-xs); margin-top: var(--sp-1); max-width: 36em; } + .wf-enabled-toggle { margin: var(--sp-3) 0 var(--sp-1); } + .wf-enabled-toggle .config-toggle-label { font-weight: 700; } + .wf-enabled-explain { margin: 0 0 var(--sp-4); max-width: 60em; } + .wf-derivation-off { color: var(--status-warn); } .wf-prov, .wf-compiled { display: inline-block; border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); font-weight: 700; white-space: nowrap; border: var(--line-width) solid var(--border); color: var(--muted); } .wf-prov-built-in { color: var(--status-info); border-color: color-mix(in srgb, var(--status-info) 40%, transparent); background: color-mix(in srgb, var(--status-info) 10%, transparent); } .wf-prov-curated { color: var(--status-ok); border-color: color-mix(in srgb, var(--status-ok) 40%, transparent); background: color-mix(in srgb, var(--status-ok) 10%, transparent); } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-library.js b/src/pkg/dashboard/static-src/js/config-workflows-library.js index b4b9a4260a..c1f05464d1 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-library.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-library.js @@ -235,7 +235,7 @@ pr: { checks: 'unknown', review: 'pending', dco: 'unknown', conflicting: false } }; } function wfSimDefaultState() { return { kind: 'pr', repo: '', facts: wfSimDefaultFacts(), result: null, error: '', busy: false, open: false }; } - const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, + const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, enabledBusy: false, ingest: { repo: '', name: '', busy: false, result: null, error: '' }, sim: wfSimDefaultState() }; @@ -314,10 +314,30 @@ }).join(''); } - function renderWorkflowLibraryTable(workflows, bindings, owner) { + // renderWorkflowEnabledToggle is the Library header's owner switch for + // workflow.enabled (PUT /api/workflows/enabled). Pure: enabled is the + // live flag from GET /api/workflows, owner gates the control, busy dims + // it while a PUT is in flight. + function renderWorkflowEnabledToggle(enabled, owner, busy) { + const on = !!enabled; + const locked = !owner || !!busy; + const explain = on + ? 'On: a bound repository\'s stages are derived by its model\'s rules and the model\'s labels are provisioned in the repository.' + : 'Off: a binding only changes the Activity Monitor\'s columns; stages are still derived by the built-in engine.'; + return '
' + + '
' + + 'Derive stages from bound models (' + yamlHint('workflow.enabled') + ')' + + '
' + + '

' + explain + '

'; + } + + function renderWorkflowLibraryTable(workflows, bindings, owner, enabled, busy) { return '
' + '

Workflow library

' + - '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when ' + yamlHint('workflow.enabled') + ' is on, how its stages are derived.

' + + renderWorkflowEnabledToggle(enabled, owner, busy) + + '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when the switch above is on, how its stages are derived.

' + '' + '' + renderWorkflowLibraryRows(workflows, bindings, owner) + '
ModelProvenanceSourceCompileS / CBound
' + '
'; @@ -783,7 +803,7 @@ } else if (_wfLib.view === 'marketplace') { body = renderWorkflowMarketplaceView(owner); } else { - body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner); + body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner, _workflowState.enabled, _wfLib.enabledBusy); } return wfLibNav(_wfLib.view) + body; } @@ -829,6 +849,45 @@ openConfigDialog('governor', null, 'Workflows'); } + // wfToggleEnabled flips workflow.enabled through PUT /api/workflows/enabled. + // Turning it on changes how every bound repository's stages are derived + // and creates labels in those repositories, so it asks first; turning it + // off is immediate (columns stay, derivation reverts to the built-in + // engine). + async function wfToggleEnabled(el) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to change workflow.enabled.', 'error'); return; } + if (_wfLib.enabledBusy) return; + const next = !_workflowState.enabled; + if (next) { + const bound = (_workflowState.bindings || []).length; + const ok = await hiveConfirm({ + title: 'Derive stages from bound models?', + message: 'Stages for ' + (bound ? bound + ' bound repositor' + (bound === 1 ? 'y' : 'ies') : 'any repository you bind') + ' will be derived by the bound model\'s rules instead of the built-in engine, and each model\'s labels are created in the repository now. Unbound repositories are unaffected. You can turn this off again at any time.', + confirmLabel: 'Turn on', + }); + if (!ok) return; + } + _wfLib.enabledBusy = true; + wfLibRerender(); + try { + const res = await fetch('/api/workflows/enabled', { method: 'PUT', headers: _inceptionAuthHeaders(), body: JSON.stringify({ enabled: next }) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + _workflowState.enabled = !!data.enabled; + if (Array.isArray(data.bindings)) _workflowState.bindings = data.bindings; + if (Array.isArray(data.warnings) && data.warnings.length) { + showToast('Derivation is on, but some labels could not be provisioned yet (the next sweep retries): ' + data.warnings.join('; '), 'error'); + } else { + showToast(_workflowState.enabled ? 'Stages are now derived from bound models.' : 'Stage derivation is back on the built-in engine; bound columns stay.'); + } + } catch (err) { + showToast('Could not change workflow.enabled: ' + err.message, 'error'); + } finally { + _wfLib.enabledBusy = false; + wfLibRerender(); + } + } + async function wfLibReload() { _workflowState.loaded = false; await loadWorkflowLibrary(); diff --git a/src/pkg/dashboard/static-src/js/config-workflows.js b/src/pkg/dashboard/static-src/js/config-workflows.js index 407bed42df..74536c106b 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows.js +++ b/src/pkg/dashboard/static-src/js/config-workflows.js @@ -60,9 +60,9 @@ const label = key + (w.description ? ' — ' + w.description : '') + (compiled ? '' : ' (draft, does not compile)'); return ``; }).join(''); - const flag = _workflowState.enabled + const flag = _workflowState.enabled || !bound ? '' - : 'derivation off'; + : 'columns only — turn on derivation in Workflows'; return `
${flag}${bound ? `bound` : 'default'}
`; } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 07b1ff15dc..e75ead995d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5053,6 +5053,10 @@ .wf-key { color: var(--text); text-decoration: none; font-weight: 700; } .wf-key:hover { text-decoration: underline; } .wf-desc { font-size: var(--fs-xs); margin-top: var(--sp-1); max-width: 36em; } + .wf-enabled-toggle { margin: var(--sp-3) 0 var(--sp-1); } + .wf-enabled-toggle .config-toggle-label { font-weight: 700; } + .wf-enabled-explain { margin: 0 0 var(--sp-4); max-width: 60em; } + .wf-derivation-off { color: var(--status-warn); } .wf-prov, .wf-compiled { display: inline-block; border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); font-weight: 700; white-space: nowrap; border: var(--line-width) solid var(--border); color: var(--muted); } .wf-prov-built-in { color: var(--status-info); border-color: color-mix(in srgb, var(--status-info) 40%, transparent); background: color-mix(in srgb, var(--status-info) 10%, transparent); } .wf-prov-curated { color: var(--status-ok); border-color: color-mix(in srgb, var(--status-ok) 40%, transparent); background: color-mix(in srgb, var(--status-ok) 10%, transparent); } @@ -62957,9 +62961,9 @@

Agent Watchdog

const label = key + (w.description ? ' — ' + w.description : '') + (compiled ? '' : ' (draft, does not compile)'); return ``; }).join(''); - const flag = _workflowState.enabled + const flag = _workflowState.enabled || !bound ? '' - : 'derivation off'; + : 'columns only — turn on derivation in Workflows'; return `
${flag}${bound ? `bound` : 'default'}
`; } @@ -63228,7 +63232,7 @@

Agent Watchdog

pr: { checks: 'unknown', review: 'pending', dco: 'unknown', conflicting: false } }; } function wfSimDefaultState() { return { kind: 'pr', repo: '', facts: wfSimDefaultFacts(), result: null, error: '', busy: false, open: false }; } - const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, + const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, enabledBusy: false, ingest: { repo: '', name: '', busy: false, result: null, error: '' }, sim: wfSimDefaultState() }; @@ -63307,10 +63311,30 @@

Agent Watchdog

}).join(''); } - function renderWorkflowLibraryTable(workflows, bindings, owner) { + // renderWorkflowEnabledToggle is the Library header's owner switch for + // workflow.enabled (PUT /api/workflows/enabled). Pure: enabled is the + // live flag from GET /api/workflows, owner gates the control, busy dims + // it while a PUT is in flight. + function renderWorkflowEnabledToggle(enabled, owner, busy) { + const on = !!enabled; + const locked = !owner || !!busy; + const explain = on + ? 'On: a bound repository\'s stages are derived by its model\'s rules and the model\'s labels are provisioned in the repository.' + : 'Off: a binding only changes the Activity Monitor\'s columns; stages are still derived by the built-in engine.'; + return '
' + + '
' + + 'Derive stages from bound models (' + yamlHint('workflow.enabled') + ')' + + '
' + + '

' + explain + '

'; + } + + function renderWorkflowLibraryTable(workflows, bindings, owner, enabled, busy) { return '
' + '

Workflow library

' + - '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when ' + yamlHint('workflow.enabled') + ' is on, how its stages are derived.

' + + renderWorkflowEnabledToggle(enabled, owner, busy) + + '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when the switch above is on, how its stages are derived.

' + '' + '' + renderWorkflowLibraryRows(workflows, bindings, owner) + '
ModelProvenanceSourceCompileS / CBound
' + '
'; @@ -63776,7 +63800,7 @@

Agent Watchdog

} else if (_wfLib.view === 'marketplace') { body = renderWorkflowMarketplaceView(owner); } else { - body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner); + body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner, _workflowState.enabled, _wfLib.enabledBusy); } return wfLibNav(_wfLib.view) + body; } @@ -63822,6 +63846,45 @@

Agent Watchdog

openConfigDialog('governor', null, 'Workflows'); } + // wfToggleEnabled flips workflow.enabled through PUT /api/workflows/enabled. + // Turning it on changes how every bound repository's stages are derived + // and creates labels in those repositories, so it asks first; turning it + // off is immediate (columns stay, derivation reverts to the built-in + // engine). + async function wfToggleEnabled(el) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to change workflow.enabled.', 'error'); return; } + if (_wfLib.enabledBusy) return; + const next = !_workflowState.enabled; + if (next) { + const bound = (_workflowState.bindings || []).length; + const ok = await hiveConfirm({ + title: 'Derive stages from bound models?', + message: 'Stages for ' + (bound ? bound + ' bound repositor' + (bound === 1 ? 'y' : 'ies') : 'any repository you bind') + ' will be derived by the bound model\'s rules instead of the built-in engine, and each model\'s labels are created in the repository now. Unbound repositories are unaffected. You can turn this off again at any time.', + confirmLabel: 'Turn on', + }); + if (!ok) return; + } + _wfLib.enabledBusy = true; + wfLibRerender(); + try { + const res = await fetch('/api/workflows/enabled', { method: 'PUT', headers: _inceptionAuthHeaders(), body: JSON.stringify({ enabled: next }) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + _workflowState.enabled = !!data.enabled; + if (Array.isArray(data.bindings)) _workflowState.bindings = data.bindings; + if (Array.isArray(data.warnings) && data.warnings.length) { + showToast('Derivation is on, but some labels could not be provisioned yet (the next sweep retries): ' + data.warnings.join('; '), 'error'); + } else { + showToast(_workflowState.enabled ? 'Stages are now derived from bound models.' : 'Stage derivation is back on the built-in engine; bound columns stay.'); + } + } catch (err) { + showToast('Could not change workflow.enabled: ' + err.message, 'error'); + } finally { + _wfLib.enabledBusy = false; + wfLibRerender(); + } + } + async function wfLibReload() { _workflowState.loaded = false; await loadWorkflowLibrary(); diff --git a/src/pkg/dashboard/workflows_library_ui_test.go b/src/pkg/dashboard/workflows_library_ui_test.go index 9d02ec9922..e7f0f5156b 100644 --- a/src/pkg/dashboard/workflows_library_ui_test.go +++ b/src/pkg/dashboard/workflows_library_ui_test.go @@ -124,3 +124,59 @@ console.log('ok'); t.Fatalf("unexpected node output: %s", out) } } + +// The Library header's workflow.enabled switch (hivecommons/hive#17620): +// wired to PUT /api/workflows/enabled with a confirm on the way on, and the +// Repos binding row's "columns only" hint when a binding exists while +// derivation is off. +func TestWorkflowsEnabledToggleWiring(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + `function renderWorkflowEnabledToggle(enabled, owner, busy)`, + `async function wfToggleEnabled(el)`, + `fetch('/api/workflows/enabled', { method: 'PUT'`, + `title: 'Derive stages from bound models?'`, + `data-action="wfToggleEnabled"`, + `renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner, _workflowState.enabled, _wfLib.enabledBusy)`, + `columns only — turn on derivation in Workflows`, + `.wf-enabled-toggle {`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing workflow.enabled toggle wiring %q", want) + } + } +} + +func TestWorkflowsEnabledToggleNode(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not installed") + } + html := indexHTML(t) + script := `const assert = require('assert'); +function esc(s) { return String(s).replace(/&/g, '&').replace(/' + p + ''; } +` + jsFunc(t, html, "renderWorkflowEnabledToggle") + ` +const off = renderWorkflowEnabledToggle(false, true, false); +assert.ok(off.includes('data-wf-enabled="off"') && off.includes('aria-checked="false"'), 'off state'); +assert.ok(off.includes('data-action="wfToggleEnabled"'), 'owner can flip'); +assert.ok(!off.includes('config-toggle-switch on'), 'switch not on'); +assert.ok(off.includes('Off: a binding only changes'), 'off explanation'); +assert.ok(off.includes('workflow.enabled'), 'names the yaml key'); + +const on = renderWorkflowEnabledToggle(true, true, false); +assert.ok(on.includes('config-toggle-switch on') && on.includes('aria-checked="true"'), 'on state'); +assert.ok(on.includes('On: a bound repository'), 'on explanation'); + +const viewer = renderWorkflowEnabledToggle(false, false, false); +assert.ok(!viewer.includes('data-action="wfToggleEnabled"') && viewer.includes('aria-disabled="true"') && viewer.includes(' disabled'), 'non-owner sees a locked switch'); +assert.ok(viewer.includes('Owner role is required'), 'non-owner title explains'); + +const busy = renderWorkflowEnabledToggle(true, true, true); +assert.ok(!busy.includes('data-action="wfToggleEnabled"') && busy.includes(' disabled'), 'busy switch is locked'); +` + cmd := exec.Command(node, "-e", script) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("node: %v\n%s", err, out) + } +} diff --git a/src/pkg/github/automerge/automerge_stage_labels_gc.go b/src/pkg/github/automerge/automerge_stage_labels_gc.go index d0d9747ea4..6861ca575b 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_gc.go +++ b/src/pkg/github/automerge/automerge_stage_labels_gc.go @@ -180,3 +180,18 @@ func (c *Engine) workflowLabelInUse(ctx context.Context, owner, repo, name strin } return res.GetTotal(), nil } + +// EnsureStageLabels provisions owner/repo's label vocabulary now — the +// built-in family plus the bound workflow model's LabelDefs — through the +// same memoised path the sweep's stage pass uses (ensureStageLabels), so a +// dashboard action that turns model-based derivation on can create the +// bound models' labels immediately instead of waiting for the next PR that +// needs a label write. The shared tracker's memo is honoured and updated: +// a repository already provisioned for the current profile hash is a +// no-op, and the sweep will not re-list it afterwards. +func (c *Engine) EnsureStageLabels(ctx context.Context, owner, repo string) error { + if c == nil || c.gh == nil { + return fmt.Errorf("stage labels: no GitHub transport") + } + return c.ensureStageLabels(ctx, owner, repo, c.stageTrackerOrShared(), c.stagePolicy().AuthorKindLabels) +} diff --git a/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go b/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go index ef0c3505e6..3247be7128 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go +++ b/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go @@ -369,3 +369,33 @@ func TestEnsureStageLabelsGCNoStoreOrRecordIsInert(t *testing.T) { t.Fatalf("unbound repo touched GC routes: %v %v", gcAPI.searches, gcAPI.deleted) } } + +// EnsureStageLabels is the dashboard's on-demand entry into the same +// memoised path (PUT /api/workflows/enabled turning derivation on). +func TestEnsureStageLabelsExportedUsesEngineTrackerAndPolicy(t *testing.T) { + ctx := context.Background() + api := &stageAPI{labels: map[string]string{}} + srv := newStageAPI(t, api) + defer srv.Close() + profile := workflowTestProfile(t, workflowmodel.LabelEntry{Name: "docs/010-requested", State: "requested"}) + e, tracker := newWorkflowStageEngine(t, srv.URL, func(string, string) *workflowmodel.Profile { return profile }) + + if err := e.EnsureStageLabels(ctx, "acme", "widget"); err != nil { + t.Fatal(err) + } + if api.lists != 1 || !slices.Contains(api.creates, "docs/010-requested") || tracker.ensuredWorkflow["acme/widget"] != profile.Hash { + t.Fatalf("lists=%d creates=%v memo=%q", api.lists, api.creates, tracker.ensuredWorkflow["acme/widget"]) + } + // The sweep's own path then finds the memo and does nothing. + if err := e.ensureStageLabels(ctx, "acme", "widget", tracker, authorkind.DefaultNames); err != nil || api.lists != 1 { + t.Fatalf("sweep after dashboard provisioning: err=%v lists=%d", err, api.lists) + } + + var nilEngine *Engine + if err := nilEngine.EnsureStageLabels(ctx, "acme", "widget"); err == nil { + t.Fatal("nil engine must report an error") + } + if err := New(nil, Options{}).EnsureStageLabels(ctx, "acme", "widget"); err == nil { + t.Fatal("engine without a transport must report an error") + } +} From 4716c47886ba6742164d4724c1ff294ac6b0d369 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 07:40:26 +0000 Subject: [PATCH 069/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.229.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.229.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-knowledge-wisprflow-source.md | 1 - changelog.d/fixed-pluk-log-dir-permission.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-knowledge-wisprflow-source.md delete mode 100644 changelog.d/fixed-pluk-log-dir-permission.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a5948e1c87..da5445321a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.229.0) + +### Added + +- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. + +### Fixed + +- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). + ## 2026-11-12 (v5.228.6) ### Fixed diff --git a/changelog.d/added-knowledge-wisprflow-source.md b/changelog.d/added-knowledge-wisprflow-source.md deleted file mode 100644 index 3e63a6018a..0000000000 --- a/changelog.d/added-knowledge-wisprflow-source.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. diff --git a/changelog.d/fixed-pluk-log-dir-permission.md b/changelog.d/fixed-pluk-log-dir-permission.md deleted file mode 100644 index 4056c1f6f9..0000000000 --- a/changelog.d/fixed-pluk-log-dir-permission.md +++ /dev/null @@ -1 +0,0 @@ -- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). From a59e6af0f4f2665dae7751d0e075a97879ebfc7b Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:40:42 +0100 Subject: [PATCH 070/223] fix: classify plain HTTP proxy forwarding failures (#17864) Hive-Run: hivecommons/hive#17859 Hive-Plan: classify repeated proxy forward failures Hive-Spec: fleet-report#runtime-classes Signed-off-by: Karibhou Co-authored-by: Karibhou Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-proxy-plain-forward-report.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/delegatedecision/issue_judgment.go | 1 + src/pkg/delegatedecision/issue_judgment_test.go | 2 +- src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 2 ++ 6 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-proxy-plain-forward-report.md diff --git a/changelog.d/fixed-proxy-plain-forward-report.md b/changelog.d/fixed-proxy-plain-forward-report.md new file mode 100644 index 0000000000..2e525cf0dd --- /dev/null +++ b/changelog.d/fixed-proxy-plain-forward-report.md @@ -0,0 +1 @@ +- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index ad7f77ae2a..fcd9a05adc 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -153,6 +153,7 @@ title is `[selfreport] : `. | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `copilot_cli_hung` | a Copilot agent showed none of the CLI's startup chrome (input prompt, help hint, login request) 180 s after launch, outside kick delivery, so the hang detector killed its session, ran bare `copilot` to capture the error and relaunched the agent (`copilot hung with no CLI prompt, running diagnostic`, `crux/agent/manager_poll.go`), or that diagnostic saw no output before its own deadline and relaunched anyway (`diagnostic: timed out waiting for copilot error output`, `crux/agent/manager_credentials.go`; the line carries the pane's `output_tail`). The key is the message, so ≥5 in 10 min counts every Copilot agent on the hive together: its Copilot agents keep failing to reach their prompt. Typical causes are a stale Copilot token the CLI hangs on (the diagnostic restores or clears it when it sees an auth error), a blocked network path to the Copilot API, or a CLI build whose startup screen the detector does not recognise (`cliReadyIndicators`, `crux/agent/pane_classify.go`). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) or of a Copilot usage-sniffed connection (`copilot sniff: upstream dial failed`, `crux/proxy/github_proxy_copilot.go`) — typically DNS or network unreachability of an agent-requested host (for example a dial timeout to `telemetry.enterprise.githubcopilot.com`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_plain_forward_failed` | the proxy could not complete an agent's plain-HTTP upstream request (`proxy: plain HTTP forward failed`, `crux/proxy/github_proxy_connect.go`), returning HTTP 502. Check the requested host and logged error, including DNS and outbound connectivity; this differs from a destination refused by the proxy policy. Log-fed: ≥5 occurrences in 10 min, clears after 20 quiet minutes | medium | | `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index e21278a332..377adbc3da 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -101,6 +101,7 @@ var selfreportOpsRemedies = map[string]string{ "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 4c20d4367e..9f0bdc0ae3 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -160,7 +160,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, + fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, } { diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 9e9b3c3c48..3946d8adb4 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -79,6 +79,7 @@ const ( ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" + ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" ClassInferenceUpstreamError = "inference_upstream_error" ClassInferenceCallerUnknown = "inference_caller_unidentified" @@ -201,6 +202,7 @@ var Classes = map[string]ClassSpec{ ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, + ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index da3b933e4a..bf9c2dafbf 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -456,6 +456,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, + {class: ClassProxyPlainForwardFailed, message: "proxy: plain HTTP forward failed"}, {class: ClassProxyPlainForwardRefused, message: "proxy: plain HTTP forward refused — destination is not globally routable"}, {class: ClassInferenceCallerUnknown, message: "inference request from unidentified caller refused"}, {class: ClassInferenceNoRoute, message: "inference no route"}, @@ -1241,6 +1242,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, "inference no route": ClassInferenceNoRoute, "inference upstream error": ClassInferenceUpstreamError, From f57a8348dc29ccadfabf1fbe8c209f5d15322ee4 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Thu, 12 Nov 2026 01:00:54 -0700 Subject: [PATCH 071/223] =?UTF-8?q?=E2=9C=A8=20Inbox:=20voice=20recorders?= =?UTF-8?q?=20as=20first-class=20input=20sources=20(Plaud=20=E2=86=92=20sp?= =?UTF-8?q?ek=20RFC=20/=20dibs=20/=20knowledge)=20(#17885)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Inbox: voice recorders as first-class input sources Add a source-agnostic Inbox (sidebar → Intelligence) that lists recordings synced by voice knowledge connectors (plaud now, wisprflow next), keyed by (source, item_id), and routes each one inside the hive to a spektacular RFC with child issues (spektacular voice --publish, or a spoke-agent fallback), a dibs draft idea, or Knowledge only — single, bulk, or by auto-routing rules applied after every sync. - pkg/inbox: Item/RouteState model, JSON store, fact-backed source, rules, spek_rfc + dibs destinations, service with sweep. - config: inbox.{sources,rules,spek,dibs} + validation. - dashboard: GET /api/inbox, POST /api/inbox/{route,dismiss}, GET|POST /api/inbox/settings, Inbox section + modals, sidebar badge. - Plaud: POST /api/knowledge/connectors/{name}/auth/complete paste fallback for hosted hives (Plaud OAuth only redirects to localhost:8199); the Integrations card now shows connection state + Open Inbox only. - docs: inbox-voice-sources.md, knowledge-connectors.md, api-reference, dashboard-sections, openapi.json. Signed-off-by: Andrew Anderson * ✨ Inbox: default spek_rfc agent fallback to first enabled agent When the hive image has no spektacular binary and inbox.spek.agent is unset, hand the RFC job to the alphabetically first enabled, unpaused agent so candidate hives can file RFCs without extra config. Signed-off-by: Andrew Anderson * 🌱 Inbox: allowlist pkg/inbox dashboard import; sidebar order test includes inbox-section Signed-off-by: Andrew Anderson * 🌱 Inbox: cover service/store/source edge cases and inbox config validation Raises pkg/inbox to 92% and pkg/config back over the 90% coverage floor. Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/added-inbox-voice-sources.md | 1 + dashboard/openapi.json | 893 ++++++++++++++++++ src/cmd/hive/boot.go | 2 + src/cmd/hive/boot_inbox.go | 82 ++ src/cmd/hive/boot_knowledge_connectors.go | 8 +- .../hive/boot_knowledge_connectors_test.go | 4 +- src/cmd/hive/main.go | 1 + src/docs/api-reference/knowledge.md | 6 + src/docs/api-reference/packs-and-acmm.md | 2 +- src/docs/dashboard-sections.md | 6 + .../dashboard-sections/workflow-and-agents.md | 14 + src/docs/documentation-map.md | 1 + src/docs/inbox-voice-sources.md | 94 ++ src/docs/knowledge-connectors.md | 16 +- .../testutil/dashboard_import_allowlist.txt | 3 + src/pkg/config/config.go | 5 +- src/pkg/config/inbox_config.go | 122 +++ .../config/inbox_config_default_agent_test.go | 51 + .../config/testdata/agent_runtime_golden.json | 4 + src/pkg/config/validate.go | 3 + src/pkg/dashboard/api.go | 1 + src/pkg/dashboard/api_inbox.go | 383 ++++++++ src/pkg/dashboard/api_inbox_test.go | 217 +++++ .../dashboard/dashboard_section_cards_test.go | 1 + .../dashboard_section_reorder_9062_test.go | 10 +- .../dashboard_section_summaries_test.go | 2 +- src/pkg/dashboard/deps.go | 4 + src/pkg/dashboard/import_count_test.go | 7 +- .../knowledge_integrations_modal_test.go | 4 +- .../dashboard/settings_parity_allowlist.txt | 6 + src/pkg/dashboard/static-src/css/inbox.css | 17 + src/pkg/dashboard/static-src/html/inbox.html | 6 + .../dashboard/static-src/html/sidebar.html | 1 + src/pkg/dashboard/static-src/js/inbox.js | 296 ++++++ src/pkg/dashboard/static-src/js/knowledge.js | 4 +- src/pkg/dashboard/static-src/js/layout.js | 2 +- src/pkg/dashboard/static-src/js/refresh.js | 3 +- src/pkg/dashboard/static-src/js/sections.js | 4 +- src/pkg/dashboard/static-src/manifest.txt | 3 + src/pkg/dashboard/static/index.html | 333 ++++++- .../dashboard/webstatic/static_index_test.go | 2 +- src/pkg/inbox/dibs.go | 113 +++ src/pkg/inbox/edges_test.go | 276 ++++++ src/pkg/inbox/factsource.go | 209 ++++ src/pkg/inbox/inbox.go | 123 +++ src/pkg/inbox/inbox_test.go | 398 ++++++++ src/pkg/inbox/rules.go | 62 ++ src/pkg/inbox/service.go | 279 ++++++ src/pkg/inbox/spek.go | 257 +++++ src/pkg/inbox/store.go | 218 +++++ src/pkg/knowledge/connector/plaud.go | 8 + .../connector/plaud_auth_complete.go | 159 ++++ .../connector/plaud_auth_complete_test.go | 79 ++ 53 files changed, 4773 insertions(+), 32 deletions(-) create mode 100644 changelog.d/added-inbox-voice-sources.md create mode 100644 src/cmd/hive/boot_inbox.go create mode 100644 src/docs/inbox-voice-sources.md create mode 100644 src/pkg/config/inbox_config.go create mode 100644 src/pkg/config/inbox_config_default_agent_test.go create mode 100644 src/pkg/dashboard/api_inbox.go create mode 100644 src/pkg/dashboard/api_inbox_test.go create mode 100644 src/pkg/dashboard/static-src/css/inbox.css create mode 100644 src/pkg/dashboard/static-src/html/inbox.html create mode 100644 src/pkg/dashboard/static-src/js/inbox.js create mode 100644 src/pkg/inbox/dibs.go create mode 100644 src/pkg/inbox/edges_test.go create mode 100644 src/pkg/inbox/factsource.go create mode 100644 src/pkg/inbox/inbox.go create mode 100644 src/pkg/inbox/inbox_test.go create mode 100644 src/pkg/inbox/rules.go create mode 100644 src/pkg/inbox/service.go create mode 100644 src/pkg/inbox/spek.go create mode 100644 src/pkg/inbox/store.go create mode 100644 src/pkg/knowledge/connector/plaud_auth_complete.go create mode 100644 src/pkg/knowledge/connector/plaud_auth_complete_test.go diff --git a/changelog.d/added-inbox-voice-sources.md b/changelog.d/added-inbox-voice-sources.md new file mode 100644 index 0000000000..a375f8cb62 --- /dev/null +++ b/changelog.d/added-inbox-voice-sources.md @@ -0,0 +1 @@ +- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 8ecb48243e..651888249f 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -37169,7 +37169,900 @@ } } } + }, + "/api/inbox": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "List inbox items (voice recordings and dictations) with their triage state", + "description": "Items are the facts written by voice knowledge connectors (`plaud`, `wisprflow`), newest first, keyed by `(source, id)`, with per-destination routing state, source connection status, candidate repositories and the auto-routing settings.", + "responses": { + "200": { + "description": "Inbox listing", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "description": "An incoming voice recording or dictation. Every item is already a Knowledge fact; the inbox only tracks triage.", + "properties": { + "source": { + "type": "string", + "description": "Connector type (plaud, wisprflow, …)" + }, + "connector": { + "type": "string" + }, + "id": { + "type": "string", + "description": "Source item id (recording id)" + }, + "title": { + "type": "string" + }, + "excerpt": { + "type": "string" + }, + "recorded_at": { + "type": "string", + "format": "date-time" + }, + "duration_ms": { + "type": "integer" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "fact_slug": { + "type": "string" + }, + "fact_path": { + "type": "string" + }, + "layer": { + "type": "string" + }, + "routes": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Per-destination routing state of an inbox item.", + "properties": { + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "filed", + "delegated", + "failed" + ] + }, + "repo": { + "type": "string", + "description": "Target repository for spek_rfc" + }, + "url": { + "type": "string", + "format": "uri", + "description": "RFC issue or dibs idea URL once filed" + }, + "children": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Child issue URLs filed by spektacular" + }, + "error": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "dismissed": { + "type": "boolean", + "description": "Kept as a Knowledge fact only" + }, + "processed": { + "type": "boolean", + "description": "Dismissed or routed to at least one destination" + } + } + } + }, + "unprocessed": { + "type": "integer" + }, + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "connector": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "connected": { + "type": "boolean" + }, + "account": { + "type": "string" + } + } + } + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Repositories this hive works on (RFC targets)" + }, + "settings": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/route": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Route an inbox item to a destination", + "description": "Owner-only. `spek_rfc` runs `spektacular voice --publish` inside the hive (or hands a prompt to the configured spoke agent) and files one RFC issue plus child issues on `repo` (default `inbox.spek.repo`); `dibs` posts the recording as a private draft idea to the configured dibs instance; `dismiss` keeps the item as a Knowledge fact only. Returns 202 with the queued state; poll `GET /api/inbox` for completion.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "source", + "item_id", + "destination" + ], + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string", + "description": "owner/repo for spek_rfc" + } + } + } + } + } + }, + "responses": { + "202": { + "description": "Route accepted", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "state": { + "type": "object", + "description": "Per-destination routing state of an inbox item.", + "properties": { + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "filed", + "delegated", + "failed" + ] + }, + "repo": { + "type": "string", + "description": "Target repository for spek_rfc" + }, + "url": { + "type": "string", + "format": "uri", + "description": "RFC issue or dibs idea URL once filed" + }, + "children": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Child issue URLs filed by spektacular" + }, + "error": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + }, + "400": { + "description": "Invalid request, unknown destination, missing repo, or destination not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown item", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "Route already queued or running", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/dismiss": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Mark inbox items as Knowledge only (or undo)", + "description": "Owner-only bulk dismiss. Dismissed items stay searchable as Knowledge facts and leave the unprocessed count; `undo: true` returns them to the inbox.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "items" + ], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + } + } + } + }, + "undo": { + "type": "boolean" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Items updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "dismissed": { + "type": "integer" + } + } + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/settings": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Read inbox auto-routing settings", + "responses": { + "200": { + "description": "Settings", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Update inbox auto-routing settings", + "description": "Owner-only. Persists `inbox.rules`, `inbox.spek.repo` and `inbox.dibs.url` to the runtime config; omitted fields are left unchanged. Rules apply to items that arrive after the change.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Settings saved", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + } + } + } + }, + "400": { + "description": "Invalid rule or repository", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/complete": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Finish a connector sign-in from pasted material (hosted hives)", + "description": "Owner-only. Plaud's OAuth client only redirects to `localhost:8199`, which a hive running in a cluster never receives. The operator pastes either the `redirect_url` the browser landed on after approving (forwarded to the in-pod callback while the login started by `/auth/connect` is still pending, 10 minutes) or the `tokens` JSON from a laptop's `~/.plaud/tokens-mcp.json`; the hive stores it and re-checks the account.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "redirect_url": { + "type": "string", + "format": "uri" + }, + "tokens": { + "type": "string", + "description": "Raw JSON contents of tokens-mcp.json" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Sign-in state after completion", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector does not support pasted completion, bad body, or no pending login", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "Upstream rejected the material", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } } + } }, "components": { "schemas": { diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 96abb844ce..91d87cde38 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -27,6 +27,7 @@ import ( "github.com/hivecommons/hive/pkg/github/automerge" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/inbox" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" "github.com/hivecommons/hive/pkg/notify" @@ -215,6 +216,7 @@ type boot struct { // bootKnowledge gitSyncer *knowledge.GitSyncer knowledgeConnectors *connector.Syncer + inboxSvc *inbox.Service knowledgePublish *connector.Mirror promotionScheduler *knowledge.PromotionScheduler quotaAccount string diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go new file mode 100644 index 0000000000..552c650911 --- /dev/null +++ b/src/cmd/hive/boot_inbox.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "time" + + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// inboxSweepTimeout bounds the auto-routing sweep run after a connector sync. +const inboxSweepTimeout = 2 * time.Minute + +// bootInbox builds the Inbox service when at least one configured knowledge +// connector is a voice source (inbox.sources, default plaud/wisprflow). +// Items are read from the connector vaults under /connectors; +// route state persists at /connector-state/_inbox.json (connector +// names cannot start with "_", so it never collides with a connector). +func (b *boot) bootInbox(baseDir string) { + sources := b.cfg.Inbox.EffectiveSources() + configured := map[string]bool{} + for _, c := range b.cfg.Knowledge.Connectors { + configured[c.Type] = true + } + var srcs []inbox.Source + for _, typ := range sources { + if configured[typ] { + srcs = append(srcs, &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ}) + } + } + if len(srcs) == 0 { + return + } + token := func(ctx context.Context, _ string) (string, error) { + if b.appAuth == nil { + return "", nil + } + return b.appAuth.Token(ctx) + } + kick := func(agent, prompt string) error { + if b.agentMgr == nil { + return fmt.Errorf("agent runtime is not available") + } + started, err := b.agentMgr.SendKickAsync(agent, prompt) + if err != nil { + return err + } + if !started { + return fmt.Errorf("agent %s did not accept the kick", agent) + } + return nil + } + store := inbox.NewStore(filepath.Join(baseDir, "connector-state", "_inbox.json")) + svc := inbox.NewService(b.cfg.Inbox, store, srcs, dashboard.NewInboxDestinations(b.cfg, token, kick), b.logger) + svc.SetReadiness(dashboard.InboxReadiness(b.cfg)) + b.inboxSvc = svc + b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) +} + +// inboxAfterSync runs the auto-routing rules once a voice connector synced. +func (b *boot) inboxAfterSync(st connector.Status) { + svc := b.inboxSvc + if svc == nil { + return + } + for _, typ := range b.cfg.Inbox.EffectiveSources() { + if typ != st.Type { + continue + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), inboxSweepTimeout) + defer cancel() + if n, err := svc.Sweep(ctx); err != nil { + b.logger.Warn("inbox sweep", "error", err, "routed", n) + } + }() + return + } +} diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index 0110c8a04b..3d9cbdc64f 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -40,7 +40,7 @@ func connectorConfigs(entries []config.KnowledgeConnector) []connector.Connector // searchable like any other vault. Cursors and status persist under // /connector-state. A nil reg uses connector.DefaultRegistry(). // Returns nil when no connectors are configured. -func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *connector.Registry, api *knowledge.KnowledgeAPI, baseDir string, logger *slog.Logger) (*connector.Syncer, error) { +func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *connector.Registry, api *knowledge.KnowledgeAPI, baseDir string, logger *slog.Logger, afterSync func(connector.Status)) (*connector.Syncer, error) { if len(entries) == 0 { return nil, nil } @@ -79,6 +79,9 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne if store := api.GetVaultStore(filepath.Join(baseDir, "connectors", st.Layer)); store != nil { store.Reindex() } + if afterSync != nil { + afterSync(st) + } }, }) } @@ -86,7 +89,7 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne // bootKnowledgeConnectors builds and starts the connector syncer. A failure // is logged and leaves connectors off; it never blocks the rest of boot. func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { - s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger) + s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil { b.logger.Warn("knowledge connectors disabled", "error", err) return @@ -99,6 +102,7 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { deps.startConnectorSyncer(b.ctx, s) } b.logger.Info("knowledge connectors started", "count", len(b.cfg.Knowledge.Connectors)) + b.bootInbox(knowledge.BaseDir()) } // knowledgeConnectorRuntime exposes the syncer to the dashboard. A nil diff --git a/src/cmd/hive/boot_knowledge_connectors_test.go b/src/cmd/hive/boot_knowledge_connectors_test.go index 048b2ea905..b0ca37dcaa 100644 --- a/src/cmd/hive/boot_knowledge_connectors_test.go +++ b/src/cmd/hive/boot_knowledge_connectors_test.go @@ -44,7 +44,7 @@ func (bootFakeConnector) Sync(_ context.Context, _ connector.Cursor, emit func(c func TestNewKnowledgeConnectorSyncer(t *testing.T) { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - if s, err := newKnowledgeConnectorSyncer(nil, nil, nil, t.TempDir(), logger); s != nil || err != nil { + if s, err := newKnowledgeConnectorSyncer(nil, nil, nil, t.TempDir(), logger, nil); s != nil || err != nil { t.Fatalf("no connectors = %v, %v; want nil, nil", s, err) } @@ -58,7 +58,7 @@ func TestNewKnowledgeConnectorSyncer(t *testing.T) { api := knowledge.NewKnowledgeAPI(nil, knowledge.KnowledgeConfig{Enabled: true, Engine: "file"}, logger) s, err := newKnowledgeConnectorSyncer([]config.KnowledgeConnector{ {Name: "guide", Type: "fake", Layer: "project"}, - }, reg, api, base, logger) + }, reg, api, base, logger, nil) if err != nil || s == nil { t.Fatalf("newKnowledgeConnectorSyncer = %v, %v", s, err) } diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 9062bf39e5..2cbba7bd59 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1146,6 +1146,7 @@ func (b *boot) wireBootClosures() { BeadSynthesizer: b.beadSynth, KnowledgeConnectors: b.knowledgeConnectorRuntime(), KnowledgePublish: b.knowledgePublishRuntime(), + Inbox: b.inboxSvc, BeadStores: b.beadStores, BeadStoreLoadFailures: b.beadStoreLoadFailures, AuditLedger: auditLedger, diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 1228e273fb..e406cdb1a3 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -65,3 +65,9 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/knowledge/connectors/{name}/auth` | Dashboard auth/session | Interactive sign-in state for a connector (Plaud OAuth): connected, account, pending, auth_url; `?refresh=1` re-checks upstream (400 for connector types without interactive sign-in) | `pkg/dashboard/api_knowledge_connector_auth.go:25` | | `POST` | `/api/knowledge/connectors/{name}/auth/connect` | Owner only | Start the connector's sign-in flow (202 with state incl. `auth_url`) | `pkg/dashboard/api_knowledge_connector_auth.go:26` | | `POST` | `/api/knowledge/connectors/{name}/auth/disconnect` | Owner only | Sign out and revoke the connector's credential upstream | `pkg/dashboard/api_knowledge_connector_auth.go:27` | +| `POST` | `/api/knowledge/connectors/{name}/auth/complete` | Owner only | Finish a sign-in from pasted material on a hosted hive: `{redirect_url}` (the `localhost:8199/auth/callback?…` URL the browser landed on, forwarded in-pod while the login is pending) or `{tokens}` (contents of `~/.plaud/tokens-mcp.json`); see [inbox-voice-sources.md](../inbox-voice-sources.md) | `pkg/dashboard/api_inbox.go:29` | +| `GET` | `/api/inbox` | Dashboard auth/session | Inbox items (voice recordings/dictations keyed by `(source, id)`) with per-destination route state, source connection status, candidate repos and auto-routing settings; see [inbox-voice-sources.md](../inbox-voice-sources.md) | `pkg/dashboard/api_inbox.go:24` | +| `POST` | `/api/inbox/route` | Owner only | Route one item: `{source, item_id, destination: spek_rfc\|dibs\|dismiss, repo?}` (202 queued; 400 unconfigured destination, 404 unknown item, 409 already queued/running) | `pkg/dashboard/api_inbox.go:25` | +| `POST` | `/api/inbox/dismiss` | Owner only | Bulk "Knowledge only": `{items: [{source, item_id}], undo?}` | `pkg/dashboard/api_inbox.go:26` | +| `GET` | `/api/inbox/settings` | Dashboard auth/session | Auto-routing rules, default RFC repo, dibs URL, destination readiness | `pkg/dashboard/api_inbox.go:27` | +| `POST` | `/api/inbox/settings` | Owner only | Update `inbox.rules` / `inbox.spek.repo` / `inbox.dibs.url` in the runtime config (omitted fields unchanged) | `pkg/dashboard/api_inbox.go:28` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index f01162b768..7dbfa576e4 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:556`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:557`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index aebdf8bdaa..22b306f356 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -184,6 +184,12 @@ The facts Hive has learned about your projects, where they came from, and how mu Full entry: [Knowledge](dashboard-sections/workflow-and-agents.md#knowledge). +## Inbox + +Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge. + +Full entry: [Inbox](dashboard-sections/workflow-and-agents.md#inbox). + ## Contributors The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them. diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index f55f9d7aa3..c49a26bbd1 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -202,6 +202,20 @@ The facts Hive has learned about your projects, where they came from, and how mu **What it is good for.** Check it when a worker keeps making the same mistake. Correct the wrong fact here. +## Inbox + +Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge. + +**What it tells you.** Every recording a voice source (Plaud, Wispr Flow) syncs is already a Knowledge fact. The Inbox lists the ones you have not triaged yet, with the source, date, length and a short excerpt. + +**How the numbers are worked out.** + +- The sidebar count and the summary count items with no destination yet and not marked "Knowledge only". +- Each item shows one pill per destination it was sent to: queued, running, filed (with a link) or failed. +- Auto-routing rules run once per new item after a sync; the first matching rule wins. + +**What it is good for.** Record an idea on your recorder, then turn it into an RFC with child issues (→ RFC), a dibs idea (→ dibs), or just keep it searchable (Knowledge only) without leaving the dashboard. See [Inbox and voice sources](../inbox-voice-sources.md). + **Example.** A fact says tests run with `make test`, but your project uses `go test`. You edit the fact. **When it appears.** Always. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 64a28aa6f7..1a5ec111b1 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -172,6 +172,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Portable AgentDefinition format](../AGENT-DEFINITION.md) — standalone YAML schema for importing/exporting agent definitions. - [How Hive injects knowledge and memory into agents](agent-knowledge-and-memory.md) — the three context layers (Hive knowledge primer, CLI conversation, CLI persistent memory), the kick-primer path and its scoping/caps, why repo `AGENTS.md` is injected rather than read natively, which per-agent home directories are fleet-shared, and what survives a kick, `/clear`, pod restart or a new agent. - [Knowledge connectors](knowledge-connectors.md) — **Owner configuration:** connect, validate, and sync external document/wiki sources; includes scope and API reference. +- [Inbox and voice sources](inbox-voice-sources.md) — triage Plaud/Wispr Flow recordings into spektacular RFCs, dibs ideas, or knowledge only; hosted-hive Plaud sign-in. - [Knowledge curator](knowledge-curator.md) — configure fact extraction, sources, vaults, document imports, code maps, and retention. - [Knowledge confidence scoring](knowledge-confidence.md) — interpret confidence badges and score explanations. - [Knowledge lifecycle states](knowledge-lifecycle.md) — approve, deprecate, or supersede facts and keep stale knowledge out of prompts. diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md new file mode 100644 index 0000000000..bb5c95792f --- /dev/null +++ b/src/docs/inbox-voice-sources.md @@ -0,0 +1,94 @@ +# Inbox and voice sources + +The **Inbox** (sidebar → Intelligence → 📥 Inbox) is where recordings and +dictations from voice sources land so you can turn them into work without +leaving the dashboard. It is source-agnostic: Plaud is the first source, +Wispr Flow (`type: wisprflow`) plugs in with the same model, and later sources +(file drop, webhook, Otter.ai, Fireflies, …) add nothing to the UI. + +Tracker RFC: *Voice recorders as first-class input sources (Inbox)* — +see the issue linked from the changelog entry. + +## Model + +- **Source** — a knowledge connector whose facts are recordings (`plaud`, + `wisprflow`). Connecting a source happens in **Knowledge → 🔌 Integrations**; + that card shows only the connection state and an **Open Inbox** button. +- **Item** — one recording, keyed `(source, item_id)` (for Plaud, the + `attr_plaud_file_id`). Every item **is already a Knowledge fact**; the Inbox + only tracks what you did with it. "📚 Fact" opens the fact in Knowledge. +- **Destination** — where an item goes: + - **→ RFC** (`spek_rfc`): the hive runs `spektacular voice --from stdin + --title --publish --repo <owner/repo>` on the transcript and + files one RFC issue plus native child issues with the hive handshake + marker. The hive's normal scanner then claims the children. If no + `spektacular` binary is on the host, the hive hands the same job to a + spoke agent as a kick prompt (state `delegated`): `inbox.spek.agent` if + set, otherwise the first enabled, unpaused agent by name. + - **→ dibs** (`dibs`): posts title, summary, transcript and the + `plaud://recordings/<id>` provenance to `POST {inbox.dibs.url}/api/ideas` + as a private draft idea (Bearer token from `inbox.dibs.token_env`). + - **Knowledge only** (`dismiss`): leaves the fact where it is and drops the + item from the unprocessed count. +- **State** per item per destination: `queued → running → filed | failed` + (or `delegated` for the agent fallback), with the RFC/idea URL and child + issue URLs. Stored in `connector-state/_inbox.json`. +- **Auto-routing rules** (⚙ Auto-routing in the Inbox, or `inbox.rules`): + `source?`, `title_prefix` or `tag`, `destination`, `repo?`. Rules run once + for each item that arrives after a connector sync; the first match wins; + items already in the Inbox are never re-routed. + +## Configuration + +```yaml +inbox: + sources: [plaud, wisprflow] # default: every configured voice connector + spek: + repo: hivecommons/hive # default RFC target + binary: spektacular # or an absolute path; empty = look on PATH + agent: "" # fallback agent; default: first enabled agent + dibs: + url: https://dibs.example.org + token_env: DIBS_TOKEN + rules: + - title_prefix: "RFC:" + destination: spek_rfc + - title_prefix: "Idea:" + destination: dibs +``` + +`inbox.rules`, `inbox.spek.repo` and `inbox.dibs.url` are editable from the +dashboard (`POST /api/inbox/settings`); the token is environment-only. + +## Hosted hives: signing in to Plaud + +Plaud's OAuth client only redirects to `http://localhost:8199/auth/callback`, +so a hive running in a cluster never receives the browser callback. The +Plaud card therefore offers **Finish sign-in by pasting**: + +1. Click **Connect Plaud**; open the authorization link and approve. +2. The browser lands on `localhost:8199/auth/callback?code=…` (a connection + error page is expected). Copy that URL and paste it into the card within + 10 minutes; the hive forwards it to the login session running in the pod. +3. Alternative: on a laptop run `npx -y @plaud-ai/mcp@latest install`, sign + in, and paste the contents of `~/.plaud/tokens-mcp.json`. + +Both go through `POST /api/knowledge/connectors/{name}/auth/complete`. +A browser-only redirect to the hive's public URL is not possible until Plaud +allows additional redirect URIs for its client. + +## Requirements on the hive image + +- Node.js ≥ 20 for the Plaud MCP server (the image ships Node 26). +- `spektacular` for → RFC: not in the image yet (tracked as a child of the + Inbox RFC); until then the agent fallback runs by default, or install it on + the host (`go install github.com/hivecommons/spektacular/cmd/spektacular@latest`). +- A GitHub token with issue-write on the target repo (the hive's app token + is injected as `GITHUB_TOKEN`). + +## API + +See [api-reference/knowledge.md](api-reference/knowledge.md) for +`GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, +`GET|POST /api/inbox/settings` and +`POST /api/knowledge/connectors/{name}/auth/complete`. diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index 6f84bf0979..e5a24b695b 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -276,12 +276,16 @@ if you do not want a sync to pull a new upstream release unreviewed. remote hive forward that port first (`ssh -L 8199:localhost:8199 user@host`) and open the URL from the forwarding machine. The login session stays open for 10 minutes; click **Check** to confirm "connected as …". - - Alternative when port-forwarding is impractical: authorize once on your - laptop with `npx -y @plaud-ai/mcp@latest install`, then copy - `~/.plaud/tokens-mcp.json` into the connector's `home` directory - (`…/connector-state/<name>/home/.plaud/`). The file is Plaud's own - token store, not a hive secret. -4. Click **Sync now**. New recordings are picked up every `interval`. + - Hosted hive (no port forward): open **Finish sign-in by pasting** on the + card and paste either the `localhost:8199/auth/callback?…` URL the + browser landed on after approving, or the contents of + `~/.plaud/tokens-mcp.json` from a laptop where `npx -y + @plaud-ai/mcp@latest install` signed in. See + [Inbox and voice sources](inbox-voice-sources.md#hosted-hives-signing-in-to-plaud). +4. Click **Sync now**. New recordings are picked up every `interval` and + appear in the **📥 Inbox**, where you file them as a spektacular RFC, send + them to dibs, or keep them as knowledge only + ([Inbox and voice sources](inbox-voice-sources.md)). ### Behaviour diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 63c7035f19..f0882b392c 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -49,6 +49,9 @@ github.com/hivecommons/hive/pkg/housestyle github.com/hivecommons/hive/pkg/hub github.com/hivecommons/hive/pkg/hubactivity github.com/hivecommons/hive/pkg/hubbackup +# pkg/inbox: Inbox triage API (hivecommons/hive#17876) — dashboard routes +# /api/inbox* wrap the inbox service; no existing seam covers it. +github.com/hivecommons/hive/pkg/inbox github.com/hivecommons/hive/pkg/inferencehealth github.com/hivecommons/hive/pkg/ioscan github.com/hivecommons/hive/pkg/knowledge diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 6c28e87550..bbeb01a2f8 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -103,7 +103,10 @@ type Config struct { Data DataConfig `yaml:"data"` Deployment DeploymentConfig `yaml:"deployment,omitempty" json:"deployment,omitempty"` Knowledge KnowledgeConfig `yaml:"knowledge"` - Hub HubConfig `yaml:"hub"` + // Inbox routes voice-recorder items (Plaud, Wispr Flow) to spek RFCs or + // dibs from the dashboard Inbox. + Inbox InboxConfig `yaml:"inbox,omitempty" json:"inbox,omitempty"` + Hub HubConfig `yaml:"hub"` // Issues tunes issue lifecycle automation. Zero value keeps the safe // defaults: close issues after their fix PR merges, and backfill hourly. Issues IssuesConfig `yaml:"issues,omitempty" json:"issues,omitempty"` diff --git a/src/pkg/config/inbox_config.go b/src/pkg/config/inbox_config.go new file mode 100644 index 0000000000..a40f0b312e --- /dev/null +++ b/src/pkg/config/inbox_config.go @@ -0,0 +1,122 @@ +package config + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Inbox destinations. `knowledge` is implicit: every item is already a +// Knowledge fact when it reaches the Inbox. +const ( + InboxDestinationSpekRFC = "spek_rfc" + InboxDestinationDibs = "dibs" + InboxDestinationDismiss = "dismiss" +) + +// DefaultInboxSources are the connector types whose facts surface as Inbox +// items when inbox.sources is empty. +var DefaultInboxSources = []string{"plaud", "wisprflow"} + +// InboxConfig is the `inbox:` block: which connector types feed the Inbox, +// how items are auto-routed, and how each destination is reached. +type InboxConfig struct { + // Sources lists connector types (plaud, wisprflow, ...) whose facts are + // Inbox items. Empty uses DefaultInboxSources. + Sources []string `yaml:"sources,omitempty" json:"sources,omitempty"` + // Rules are evaluated in order against every new item; the first match + // routes it automatically. + Rules []InboxRule `yaml:"rules,omitempty" json:"rules,omitempty"` + Spek InboxSpek `yaml:"spek,omitempty" json:"spek,omitempty"` + Dibs InboxDibs `yaml:"dibs,omitempty" json:"dibs,omitempty"` +} + +// InboxRule routes items whose source and title/tag match. +type InboxRule struct { + // Source restricts the rule to one connector type; empty matches all. + Source string `yaml:"source,omitempty" json:"source,omitempty"` + // TitlePrefix matches case-insensitively against the item title. + TitlePrefix string `yaml:"title_prefix,omitempty" json:"title_prefix,omitempty"` + // Tag matches any item tag exactly (case-insensitive). + Tag string `yaml:"tag,omitempty" json:"tag,omitempty"` + // Destination is spek_rfc, dibs or dismiss. + Destination string `yaml:"destination" json:"destination"` + // Repo (owner/repo) overrides inbox.spek.repo for spek_rfc. + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` +} + +// InboxSpek configures the spek_rfc destination. +type InboxSpek struct { + // Repo is the default owner/repo RFCs are filed to. + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` + // Binary is the spektacular executable (default runs.spektacular.binary). + Binary string `yaml:"binary,omitempty" json:"binary,omitempty"` + // Agent names a hive agent to kick with a spek-new/publish prompt when + // the binary is unavailable in the hive image. Empty picks the first + // enabled, unpaused agent by name (see InboxSpekAgent). + Agent string `yaml:"agent,omitempty" json:"agent,omitempty"` +} + +// InboxSpekAgent returns the agent the spek_rfc destination falls back to +// when the spektacular binary is absent: inbox.spek.agent if set, else the +// alphabetically first enabled, unpaused agent, else "". +func (c *Config) InboxSpekAgent() string { + if a := strings.TrimSpace(c.Inbox.Spek.Agent); a != "" { + return a + } + names := make([]string, 0, len(c.Agents)) + for name, a := range c.Agents { + if a.Enabled && !a.Paused { + names = append(names, name) + } + } + sort.Strings(names) + if len(names) == 0 { + return "" + } + return names[0] +} + +// InboxDibs configures the dibs destination. +type InboxDibs struct { + // URL is the dibs instance base URL. + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // TokenEnv names the environment variable holding the hub bearer token. + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` +} + +var inboxRepoRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`) + +// EffectiveSources returns Sources or DefaultInboxSources. +func (c InboxConfig) EffectiveSources() []string { + if len(c.Sources) == 0 { + return append([]string(nil), DefaultInboxSources...) + } + return c.Sources +} + +// ValidateInbox rejects unknown destinations, malformed repos and rules +// without a filter (a filter-less rule would route every item). +func ValidateInbox(c InboxConfig) error { + if c.Spek.Repo != "" && !inboxRepoRe.MatchString(c.Spek.Repo) { + return fmt.Errorf("inbox.spek.repo %q: must be owner/repo", c.Spek.Repo) + } + if c.Dibs.URL != "" && !strings.HasPrefix(c.Dibs.URL, "http://") && !strings.HasPrefix(c.Dibs.URL, "https://") { + return fmt.Errorf("inbox.dibs.url %q: must start with http:// or https://", c.Dibs.URL) + } + for i, r := range c.Rules { + switch r.Destination { + case InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss: + default: + return fmt.Errorf("inbox.rules[%d].destination %q: must be %s, %s or %s", i, r.Destination, InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss) + } + if strings.TrimSpace(r.TitlePrefix) == "" && strings.TrimSpace(r.Tag) == "" { + return fmt.Errorf("inbox.rules[%d]: title_prefix or tag is required", i) + } + if r.Repo != "" && !inboxRepoRe.MatchString(r.Repo) { + return fmt.Errorf("inbox.rules[%d].repo %q: must be owner/repo", i, r.Repo) + } + } + return nil +} diff --git a/src/pkg/config/inbox_config_default_agent_test.go b/src/pkg/config/inbox_config_default_agent_test.go new file mode 100644 index 0000000000..a3193c4e13 --- /dev/null +++ b/src/pkg/config/inbox_config_default_agent_test.go @@ -0,0 +1,51 @@ +package config + +import "testing" + +func TestInboxSpekAgentDefaultsToFirstEnabledAgent(t *testing.T) { + c := &Config{AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "zed": {Enabled: true}, + "alpha": {Enabled: true, Paused: true}, + "bravo": {Enabled: true}, + "charly": {Enabled: false}, + }}} + if got := c.InboxSpekAgent(); got != "bravo" { + t.Fatalf("default agent = %q, want bravo (first enabled, unpaused)", got) + } + c.Inbox.Spek.Agent = " custom " + if got := c.InboxSpekAgent(); got != "custom" { + t.Fatalf("explicit agent = %q, want custom", got) + } + if got := (&Config{}).InboxSpekAgent(); got != "" { + t.Fatalf("no agents should give empty, got %q", got) + } +} + +func TestInboxEffectiveSourcesAndValidate(t *testing.T) { + if got := (InboxConfig{}).EffectiveSources(); len(got) != len(DefaultInboxSources) || got[0] != DefaultInboxSources[0] { + t.Fatalf("default sources = %v", got) + } + if got := (InboxConfig{Sources: []string{"wisprflow"}}).EffectiveSources(); len(got) != 1 || got[0] != "wisprflow" { + t.Fatalf("explicit sources = %v", got) + } + ok := InboxConfig{ + Spek: InboxSpek{Repo: "o/r"}, + Dibs: InboxDibs{URL: "https://dibs.example"}, + Rules: []InboxRule{{TitlePrefix: "RFC:", Destination: InboxDestinationSpekRFC, Repo: "o/r"}, {Tag: "idea", Destination: InboxDestinationDibs}}, + } + if err := ValidateInbox(ok); err != nil { + t.Fatalf("valid config rejected: %v", err) + } + bad := map[string]InboxConfig{ + "spek repo": {Spek: InboxSpek{Repo: "not-a-repo"}}, + "dibs url": {Dibs: InboxDibs{URL: "dibs.example"}}, + "rule destination": {Rules: []InboxRule{{TitlePrefix: "x", Destination: "email"}}}, + "rule filter": {Rules: []InboxRule{{Destination: InboxDestinationDismiss}}}, + "rule repo": {Rules: []InboxRule{{TitlePrefix: "x", Destination: InboxDestinationSpekRFC, Repo: "nope"}}}, + } + for name, c := range bad { + if err := ValidateInbox(c); err == nil { + t.Errorf("%s: expected an error", name) + } + } +} diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index a70784ec8f..f61ae46f36 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -570,6 +570,10 @@ "OnError": "" } }, + "inbox": { + "spek": {}, + "dibs": {} + }, "Hub": { "Enabled": false, "URL": "https://hive.hivecommons.dev", diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8aab8ad642..8812f3e6cf 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -208,6 +208,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { for _, w := range advisorWarnings { log.Printf("WARNING: %s", w) } + if err := ValidateInbox(c.Inbox); err != nil { + return err + } if err := ValidateKnowledgeConnectors(c.Knowledge.Connectors); err != nil { return err } diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index af8380cc4a..ceebd002c0 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -549,6 +549,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // Knowledge connector status + manual sync (#11069, api_knowledge_connectors.go). s.registerKnowledgeConnectorRoutes() s.registerKnowledgeConnectorAuthRoutes() + s.registerInboxRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) // Per-agent Variables tab (#11458, api_variables.go). s.registerAgentVariableRoutes() diff --git a/src/pkg/dashboard/api_inbox.go b/src/pkg/dashboard/api_inbox.go new file mode 100644 index 0000000000..1ca6dc14bd --- /dev/null +++ b/src/pkg/dashboard/api_inbox.go @@ -0,0 +1,383 @@ +package dashboard + +import ( + "context" + "errors" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// inboxListTimeout bounds one Inbox listing (reads connector facts). +const inboxListTimeout = 20 * time.Second + +// Inbox (#voice-sources): source-agnostic triage of voice-recorder items +// that knowledge connectors ingest, with routes to spek RFCs and dibs. +func (s *Server) registerInboxRoutes() { + s.mux.HandleFunc("GET /api/inbox", s.handleInboxList) + s.mux.HandleFunc("POST /api/inbox/route", s.handleInboxRoute) + s.mux.HandleFunc("POST /api/inbox/dismiss", s.handleInboxDismiss) + s.mux.HandleFunc("GET /api/inbox/settings", s.handleInboxSettingsGet) + s.mux.HandleFunc("POST /api/inbox/settings", s.handleInboxSettingsPost) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/complete", s.handleKnowledgeConnectorAuthComplete) +} + +// InboxSourceStatus is one configured source as the Inbox header shows it. +type InboxSourceStatus struct { + Type string `json:"type"` + Connector string `json:"connector,omitempty"` + // Configured is true when a knowledge.connectors entry of this type exists. + Configured bool `json:"configured"` + // Connected mirrors the connector's AuthState when it signs in interactively. + Connected bool `json:"connected"` + Account string `json:"account,omitempty"` +} + +// InboxListResponse is GET /api/inbox. +type InboxListResponse struct { + Items []inbox.Item `json:"items"` + Unprocessed int `json:"unprocessed"` + Sources []InboxSourceStatus `json:"sources"` + // Repos are the hive's configured repos for the RFC repo picker. + Repos []string `json:"repos"` + Settings inbox.Settings `json:"settings"` + Error string `json:"error,omitempty"` +} + +func (s *Server) inboxService(w http.ResponseWriter) (*inbox.Service, bool) { + if s == nil || s.deps == nil || s.deps.Inbox == nil { + jsonError(w, "inbox is not available: configure a voice knowledge connector (plaud, wisprflow)", http.StatusNotFound) + return nil, false + } + return s.deps.Inbox, true +} + +func (s *Server) handleInboxList(w http.ResponseWriter, r *http.Request) { + svc, ok := s.inboxService(w) + if !ok { + return + } + ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) + defer cancel() + items, err := svc.Items(ctx) + resp := InboxListResponse{ + Items: items, + Unprocessed: inbox.Unprocessed(items), + Sources: s.inboxSources(ctx, svc.Config().EffectiveSources()), + Repos: s.inboxRepos(), + Settings: svc.Settings(), + } + if resp.Items == nil { + resp.Items = []inbox.Item{} + } + if err != nil { + resp.Error = err.Error() + } + jsonResponse(w, resp) +} + +// inboxSources pairs each source type with its connector's cached auth state. +func (s *Server) inboxSources(ctx context.Context, types []string) []InboxSourceStatus { + out := make([]InboxSourceStatus, 0, len(types)) + rt := s.knowledgeConnectors() + var statuses []connector.Status + if rt != nil { + statuses = rt.Statuses() + } + lookup, _ := rt.(knowledgeConnectorLookup) + for _, typ := range types { + st := InboxSourceStatus{Type: typ} + for _, cs := range statuses { + if cs.Type != typ { + continue + } + st.Configured, st.Connector = true, cs.Name + if lookup != nil { + if c, ok := lookup.Connector(cs.Name); ok { + if auth, ok := c.(connector.Authenticator); ok { + a := auth.AuthStatus(ctx, false) + st.Connected, st.Account = a.Connected, a.Account + } + } + } + break + } + out = append(out, st) + } + return out +} + +func (s *Server) inboxRepos() []string { + if s.deps == nil || s.deps.Config == nil { + return []string{} + } + repos := append([]string(nil), s.deps.Config.Project.Repos...) + if def := s.deps.Config.Inbox.Spek.Repo; def != "" { + found := false + for _, r := range repos { + if r == def { + found = true + } + } + if !found { + repos = append([]string{def}, repos...) + } + } + if repos == nil { + repos = []string{} + } + return repos +} + +type inboxRouteRequest struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + Destination string `json:"destination"` + Repo string `json:"repo,omitempty"` +} + +// InboxRouteResponse is POST /api/inbox/route. +type InboxRouteResponse struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + Destination string `json:"destination"` + State inbox.RouteState `json:"state"` +} + +// handleInboxRoute (owner) queues one item for one destination; progress is +// read back from GET /api/inbox. +func (s *Server) handleInboxRoute(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxRouteRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + req.Source, req.ItemID, req.Destination, req.Repo = strings.TrimSpace(req.Source), strings.TrimSpace(req.ItemID), strings.TrimSpace(req.Destination), strings.TrimSpace(req.Repo) + if req.Source == "" || req.ItemID == "" || req.Destination == "" { + jsonError(w, "source, item_id and destination are required", http.StatusBadRequest) + return + } + if req.Repo != "" && !inboxRepoOK(req.Repo) { + jsonError(w, "repo must be owner/repo", http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) + defer cancel() + it, found, err := svc.Find(ctx, req.Source, req.ItemID) + if !found { + msg := "unknown inbox item" + if err != nil { + msg += ": " + err.Error() + } + jsonError(w, msg, http.StatusNotFound) + return + } + st, err := svc.Route(ctx, it, inbox.Request{Destination: req.Destination, Repo: req.Repo}) + switch { + case errors.Is(err, inbox.ErrRouteBusy): + jsonError(w, "that route is already queued or running", http.StatusConflict) + return + case errors.Is(err, inbox.ErrUnknownDestination): + jsonError(w, err.Error(), http.StatusBadRequest) + return + case err != nil: + jsonError(w, "route failed: "+err.Error(), http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "inbox_route", auditDetail("source", req.Source, "item", req.ItemID, "destination", req.Destination, "repo", req.Repo), "") + jsonStatusResponse(w, http.StatusAccepted, InboxRouteResponse{Source: req.Source, ItemID: req.ItemID, Destination: req.Destination, State: st}) +} + +func inboxRepoOK(repo string) bool { + owner, name, ok := strings.Cut(repo, "/") + return ok && owner != "" && name != "" && !strings.ContainsAny(repo, " \t\n") +} + +type inboxDismissRequest struct { + Items []struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + } `json:"items"` + Undo bool `json:"undo,omitempty"` +} + +// InboxDismissResponse is POST /api/inbox/dismiss. +type InboxDismissResponse struct { + Dismissed int `json:"dismissed"` +} + +// handleInboxDismiss (owner) marks items "knowledge only" (bulk). +func (s *Server) handleInboxDismiss(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxDismissRequest + if err := decodeBody(r, &req); err != nil || len(req.Items) == 0 { + jsonError(w, "items[] is required", http.StatusBadRequest) + return + } + n := 0 + for _, it := range req.Items { + if it.Source == "" || it.ItemID == "" { + continue + } + if err := svc.Dismiss(it.Source, it.ItemID, !req.Undo); err != nil { + jsonError(w, "dismiss failed: "+err.Error(), http.StatusInternalServerError) + return + } + n++ + } + s.auditFromRequest(r, "inbox_dismiss", auditDetail("count", strconv.Itoa(n)), "") + jsonResponse(w, InboxDismissResponse{Dismissed: n}) +} + +func (s *Server) handleInboxSettingsGet(w http.ResponseWriter, r *http.Request) { + svc, ok := s.inboxService(w) + if !ok { + return + } + jsonResponse(w, svc.Settings()) +} + +type inboxSettingsRequest struct { + Rules *[]config.InboxRule `json:"rules,omitempty"` + SpekRepo *string `json:"spek_repo,omitempty"` + DibsURL *string `json:"dibs_url,omitempty"` +} + +// handleInboxSettingsPost (owner) updates inbox.rules / inbox.spek.repo / +// inbox.dibs.url and persists hive.yaml. +func (s *Server) handleInboxSettingsPost(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + if s.deps.Config == nil { + jsonError(w, "config is not available", http.StatusServiceUnavailable) + return + } + var req inboxSettingsRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + next := s.deps.Config.Inbox + if req.Rules != nil { + next.Rules = *req.Rules + } + if req.SpekRepo != nil { + next.Spek.Repo = strings.TrimSpace(*req.SpekRepo) + } + if req.DibsURL != nil { + next.Dibs.URL = strings.TrimRight(strings.TrimSpace(*req.DibsURL), "/") + } + if err := config.ValidateInbox(next); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + s.deps.Config.Inbox = next + svc.UpdateConfig(next) + if err := s.saveConfig(); err != nil { + jsonError(w, "saving config: "+err.Error(), http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "inbox_settings", auditDetail("rules", strconv.Itoa(len(next.Rules)), "spek_repo", next.Spek.Repo, "dibs_url", next.Dibs.URL), "") + jsonResponse(w, svc.Settings()) +} + +// handleKnowledgeConnectorAuthComplete (owner) finishes a sign-in whose +// OAuth callback cannot reach a hosted hive: the operator pastes either the +// redirect URL the browser landed on or the vendor token file contents. +func (s *Server) handleKnowledgeConnectorAuthComplete(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + completer, ok := auth.(connector.AuthCompleter) + if !ok { + jsonError(w, "this connector does not support pasted sign-in completion", http.StatusBadRequest) + return + } + var comp connector.AuthCompletion + if err := decodeBody(r, &comp); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := completer.CompleteAuth(ctx, comp) + if err != nil { + jsonError(w, err.Error(), http.StatusBadGateway) + return + } + mode := "redirect_url" + if comp.Tokens != "" { + mode = "tokens" + } + s.auditFromRequest(r, "knowledge_connector_auth_complete", auditDetail("connector", name, "mode", mode), "") + jsonResponse(w, st) +} + +// NewInboxDestinations builds the spek_rfc and dibs destinations from +// config; token minting and agent kicks come from the dashboard deps. +func NewInboxDestinations(cfg *config.Config, token func(ctx context.Context, repo string) (string, error), kick inbox.Kicker) []inbox.Destination { + binary := cfg.Inbox.Spek.Binary + if binary == "" { + binary = cfg.Runs.Spektacular.BinaryOrDefault() + } + spek := &inbox.SpekDestination{ + Binary: binary, + DefaultRepo: cfg.Inbox.Spek.Repo, + Token: token, + Agent: cfg.InboxSpekAgent(), + Kick: kick, + } + dibs := &inbox.DibsDestination{URL: cfg.Inbox.Dibs.URL} + if env := cfg.Inbox.Dibs.TokenEnv; env != "" { + dibs.Token = os.Getenv(env) + } + return []inbox.Destination{spek, dibs} +} + +// InboxReadiness returns the probes the settings view reports. +func InboxReadiness(cfg *config.Config) (spek func() bool, dibs func() bool) { + binary := cfg.Inbox.Spek.Binary + if binary == "" { + binary = cfg.Runs.Spektacular.BinaryOrDefault() + } + spek = func() bool { + if _, err := exec.LookPath(binary); err == nil { + return true + } + return cfg.InboxSpekAgent() != "" + } + dibs = func() bool { + return cfg.Inbox.Dibs.TokenEnv != "" && os.Getenv(cfg.Inbox.Dibs.TokenEnv) != "" + } + return spek, dibs +} diff --git a/src/pkg/dashboard/api_inbox_test.go b/src/pkg/dashboard/api_inbox_test.go new file mode 100644 index 0000000000..150f36c188 --- /dev/null +++ b/src/pkg/dashboard/api_inbox_test.go @@ -0,0 +1,217 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +const inboxTestFact = `--- +title: RFC: Voice inbox +type: reference +layer: personal +status: active +source: plaud +connector: plaud +source_id: rec-1 +attr_plaud_file_id: rec-1 +attr_recorded_at: 2026-11-10T08:58:00Z +synthesized: 2026-11-10T08:58:00Z +--- + +## Summary (Plaud) + +- ship the inbox + +## Transcript + +Hello hive +` + +type inboxTestDest struct { + name string + got []inbox.Request + err error +} + +func (d *inboxTestDest) Name() string { return d.name } +func (d *inboxTestDest) Deliver(_ context.Context, _ inbox.Item, _ inbox.Content, req inbox.Request) (inbox.Result, error) { + d.got = append(d.got, req) + if d.err != nil { + return inbox.Result{}, d.err + } + return inbox.Result{URL: "https://github.com/o/r/issues/1", Children: []string{"https://github.com/o/r/issues/2"}}, nil +} + +func newInboxTestService(t *testing.T, cfg config.InboxConfig, dests ...inbox.Destination) *inbox.Service { + t.Helper() + base := t.TempDir() + dir := filepath.Join(base, "connectors", "personal") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "plaud-plaud-rec-1.md"), []byte(inboxTestFact), 0o644); err != nil { + t.Fatal(err) + } + src := &inbox.FactSource{ConnectorsDir: filepath.Join(base, "connectors"), SourceType: "plaud"} + svc := inbox.NewService(cfg, inbox.NewStore(filepath.Join(base, "inbox.json")), []inbox.Source{src}, dests, nil) + svc.Run = func(fn func()) { fn() } + return svc +} + +func decodeInboxList(t *testing.T, body []byte) InboxListResponse { + t.Helper() + var resp InboxListResponse + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return resp +} + +func TestInboxEndpoints(t *testing.T) { + s, deps := apiServer(t) + if rec := doGet(s, "/api/inbox"); rec.Code != http.StatusNotFound { + t.Fatalf("inbox without service = %d, want 404", rec.Code) + } + spek := &inboxTestDest{name: inbox.DestSpekRFC} + dibs := &inboxTestDest{name: inbox.DestDibs, err: errors.New("dibs down")} + deps.Config.Inbox = config.InboxConfig{Spek: config.InboxSpek{Repo: "myorg/rfcs"}} + deps.Inbox = newInboxTestService(t, deps.Config.Inbox, spek, dibs) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &authConn{state: connector.AuthState{Connected: true, Account: "andy@example.com"}}) + + rec := doGet(s, "/api/inbox") + if rec.Code != http.StatusOK { + t.Fatalf("list = %d: %s", rec.Code, rec.Body.String()) + } + resp := decodeInboxList(t, rec.Body.Bytes()) + if len(resp.Items) != 1 || resp.Items[0].ID != "rec-1" || resp.Items[0].FactSlug != "plaud-plaud-rec-1" || resp.Unprocessed != 1 { + t.Fatalf("list = %+v", resp) + } + if len(resp.Repos) != 2 || resp.Repos[0] != "myorg/rfcs" || resp.Repos[1] != "repo1" { + t.Fatalf("repos = %v", resp.Repos) + } + if len(resp.Sources) != 2 || resp.Sources[0].Type != "plaud" || resp.Sources[0].Configured || resp.Settings.SpekRepo != "myorg/rfcs" { + t.Fatalf("sources/settings = %+v %+v", resp.Sources, resp.Settings) + } + + if rec := doPostNoOwner(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc"}); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner route = %d, want 403", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "nope", "destination": "spek_rfc"}); rec.Code != http.StatusNotFound { + t.Fatalf("unknown item = %d", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc", "repo": "bad repo"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bad repo = %d", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "nowhere"}); rec.Code != http.StatusBadRequest { + t.Fatalf("unknown destination = %d: %s", rec.Code, rec.Body.String()) + } + rec = doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc"}) + if rec.Code != http.StatusAccepted { + t.Fatalf("route = %d: %s", rec.Code, rec.Body.String()) + } + if len(spek.got) != 1 || spek.got[0].Repo != "myorg/rfcs" { + t.Fatalf("spek requests = %+v (default repo must apply)", spek.got) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + r := resp.Items[0].Routes[inbox.DestSpekRFC] + if r == nil || r.Status != inbox.StatusFiled || r.URL != "https://github.com/o/r/issues/1" || len(r.Children) != 1 || resp.Unprocessed != 0 { + t.Fatalf("spek route = %+v unprocessed=%d", r, resp.Unprocessed) + } + rec = doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "dibs"}) + if rec.Code != http.StatusAccepted { + t.Fatalf("dibs route = %d: %s", rec.Code, rec.Body.String()) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + if d := resp.Items[0].Routes[inbox.DestDibs]; d == nil || d.Status != inbox.StatusFailed || d.Error != "dibs down" { + t.Fatalf("dibs route = %+v", d) + } + + rec = doPost(s, "/api/inbox/dismiss", map[string]any{"items": []map[string]string{{"source": "plaud", "item_id": "rec-1"}}}) + if rec.Code != http.StatusOK { + t.Fatalf("dismiss = %d: %s", rec.Code, rec.Body.String()) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + if !resp.Items[0].Dismissed { + t.Fatalf("item not dismissed: %+v", resp.Items[0]) + } +} + +func TestInboxSettingsPersistRules(t *testing.T) { + s, deps := apiServer(t) + deps.Inbox = newInboxTestService(t, config.InboxConfig{}) + if rec := doPost(s, "/api/inbox/settings", map[string]any{"rules": []map[string]string{{"destination": "spek_rfc"}}}); rec.Code != http.StatusBadRequest { + t.Fatalf("rule without filter = %d, want 400: %s", rec.Code, rec.Body.String()) + } + if rec := doPost(s, "/api/inbox/settings", map[string]any{"spek_repo": "not-a-repo"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bad repo = %d, want 400", rec.Code) + } + rec := doPost(s, "/api/inbox/settings", map[string]any{ + "rules": []map[string]string{{"source": "plaud", "title_prefix": "RFC:", "destination": "spek_rfc", "repo": "o/r"}}, + "spek_repo": "o/r", + "dibs_url": "https://dibs.example/", + }) + if rec.Code != http.StatusOK { + t.Fatalf("settings = %d: %s", rec.Code, rec.Body.String()) + } + var st inbox.Settings + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if len(st.Rules) != 1 || st.Rules[0].TitlePrefix != "RFC:" || st.SpekRepo != "o/r" || st.DibsURL != "https://dibs.example" { + t.Fatalf("settings = %+v", st) + } + if deps.Config.Inbox.Spek.Repo != "o/r" || len(deps.Config.Inbox.Rules) != 1 || deps.Inbox.Config().Spek.Repo != "o/r" { + t.Fatalf("config not updated: %+v", deps.Config.Inbox) + } + if rec := doGet(s, "/api/inbox/settings"); rec.Code != http.StatusOK { + t.Fatalf("settings get = %d", rec.Code) + } +} + +// completerConn is an Authenticator that also accepts pasted completions. +type completerConn struct { + authConn + got connector.AuthCompletion +} + +func (c *completerConn) CompleteAuth(_ context.Context, comp connector.AuthCompletion) (connector.AuthState, error) { + c.got = comp + if comp.Tokens == "" && comp.RedirectURL == "" { + return c.state, errors.New("nothing pasted") + } + c.state = connector.AuthState{Connected: true, Account: "andy@example.com", CheckedAt: time.Now()} + return c.state, nil +} + +func TestKnowledgeConnectorAuthComplete(t *testing.T) { + s, deps := apiServer(t) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &authConn{}) + if rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"tokens": "{}"}); rec.Code != http.StatusBadRequest { + t.Fatalf("non-completer = %d, want 400: %s", rec.Code, rec.Body.String()) + } + conn := &completerConn{} + deps.KnowledgeConnectors = newTestConnectorSyncer(t, conn) + if rec := doPostNoOwner(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"tokens": "{}"}); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner = %d, want 403", rec.Code) + } + if rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{}); rec.Code != http.StatusBadGateway { + t.Fatalf("empty completion = %d, want 502: %s", rec.Code, rec.Body.String()) + } + rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"redirect_url": "http://localhost:8199/auth/callback?code=x&state=y"}) + if rec.Code != http.StatusOK { + t.Fatalf("complete = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Connected || conn.got.RedirectURL == "" { + t.Fatalf("state = %+v got = %+v", st, conn.got) + } +} diff --git a/src/pkg/dashboard/dashboard_section_cards_test.go b/src/pkg/dashboard/dashboard_section_cards_test.go index 11d6d11f5d..7fa8a5d0ac 100644 --- a/src/pkg/dashboard/dashboard_section_cards_test.go +++ b/src/pkg/dashboard/dashboard_section_cards_test.go @@ -274,6 +274,7 @@ func TestDashboardSectionRenderersRefreshSharedShell(t *testing.T) { "function renderFAQ()": "faq-section", "function renderInception()": "inception-section", "function renderKnowledge()": "knowledge-section", + "function renderInbox()": "inbox-section", "function renderDebugSection()": "debug-section", "function renderLogs()": "logs-section", } diff --git a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go index 8ee1af1a9d..04bf41a306 100644 --- a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go +++ b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go @@ -9,14 +9,14 @@ import ( ) var dashboardLayoutIDs9062 = []string{ - "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "review-queue-section", + "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section", "faq-section", } var dashboardDefaultNavOrder9062 = []string{ - "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "review-queue-section", + "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "faq-section", } @@ -26,7 +26,7 @@ var dashboardDefaultLayoutOrder9062 = append(append(append([]string{}, dashboard "faq-section", ) -const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` +const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` func jsArray9062(ids []string) string { var b strings.Builder @@ -165,7 +165,7 @@ func TestDashboardLayoutNormalizeFutureUnknownAndMissing(t *testing.T) { html := indexHTML(t) script := dashboardLayoutPreamble9062() + jsFunc(t, html, "dashboardLayoutAllIds") + "\n" + jsFunc(t, html, "dashboardLayoutNormalize") + ` const got = dashboardLayoutNormalize({v:99, main:['faq-section','beads-section','unknown','governor','faq-section']}); -const want = ` + jsArray9062([]string{"faq-section", "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section"}) + `; +const want = ` + jsArray9062([]string{"faq-section", "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section"}) + `; if (got.v !== 1) throw new Error('version not normalized: '+got.v); if (got.main.includes('beads-section')) throw new Error('legacy beads section survived normalization: '+JSON.stringify(got.main)); if (JSON.stringify(got.main) !== JSON.stringify(want)) throw new Error('normalized order '+JSON.stringify(got.main)); @@ -287,7 +287,7 @@ if (middle !== 'nous-section,logs-section,agents-section,agent-activity-section, if (new Set(got).size !== got.length) throw new Error('duplicate ids: '+JSON.stringify(got)); const moved = dashboardLayoutNormalize({v:1, main:['faq-section','cost-panel','overview-section']}).main; if (moved[0] !== 'faq-section') throw new Error('operator-moved FAQ not respected: '+JSON.stringify(moved)); -if (moved.length !== 21) throw new Error('missing sections not restored: '+moved.length); +if (moved.length !== 22) throw new Error('missing sections not restored: '+moved.length); ` cmd := exec.Command(node, "-e", script) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/src/pkg/dashboard/dashboard_section_summaries_test.go b/src/pkg/dashboard/dashboard_section_summaries_test.go index 53f124738c..e1f1a7f317 100644 --- a/src/pkg/dashboard/dashboard_section_summaries_test.go +++ b/src/pkg/dashboard/dashboard_section_summaries_test.go @@ -39,7 +39,7 @@ func TestDashboardSectionCardsHaveCollapsedSummaries(t *testing.T) { "overview-section", "advisory-section", "hive-advice-section", "fleet-report-section", "acmm-reco-section", "lifecycle-section", "pr-throughput-section", "repos-section", "acmm-eval-section", "approvals-section", "audit-section", - "review-queue-section", "nous-section", "inception-section", "knowledge-section", + "review-queue-section", "nous-section", "inception-section", "knowledge-section", "inbox-section", "contributors-section", "debug-section", "logs-section", "agents-section", "faq-section", } for _, id := range refreshRequired { diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index e0bff6c556..8ed04dbb1a 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -25,6 +25,7 @@ import ( "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hivestate" "github.com/hivecommons/hive/pkg/hooks" + "github.com/hivecommons/hive/pkg/inbox" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/nous" "github.com/hivecommons/hive/pkg/review/autoscale" @@ -109,6 +110,9 @@ type Dependencies struct { // KnowledgePublish is the running knowledge.publish mirror; nil when // publishing is not configured or failed to start. KnowledgePublish KnowledgePublishRuntime + // Inbox is the voice-source triage service; nil when no voice + // connector is configured. + Inbox *inbox.Service // RotationMgr is the provider-rotation headroom reporter (RFC #3958). Nil // when rotation is disabled; the headroom endpoint then reports // enabled=false. diff --git a/src/pkg/dashboard/import_count_test.go b/src/pkg/dashboard/import_count_test.go index d2e43b1707..5274543e12 100644 --- a/src/pkg/dashboard/import_count_test.go +++ b/src/pkg/dashboard/import_count_test.go @@ -66,8 +66,11 @@ func TestDashboardInternalImportCountRatchet(t *testing.T) { // shared with the scheduler's kick injection (hivecommons/hive#17244), // plus pkg/workflowmodel/store, the spoke-local library of workflow // models behind the /api/workflows routes and the tracker's per-repo - // column block (RFC hivecommons/hive#17620, phase 2). - const maxDashboardInternalImports = 60 + // column block (RFC hivecommons/hive#17620, phase 2), plus pkg/inbox, + // the source-agnostic voice-recording triage service (items keyed by + // source + item id, spek_rfc/dibs destinations) behind the /api/inbox + // routes and the Inbox dashboard section. + const maxDashboardInternalImports = 61 entries, err := os.ReadDir(".") if err != nil { diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 2c3cb41497..8dc4184b81 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -155,7 +155,9 @@ func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { for _, want := range []string{ `'plaud': { label: 'Plaud voice notes'`, `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, - `if (type === 'plaud') html += kbPlaudAuthLine(c.name);`, + `if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, + `data-action="kbPlaudPasteComplete"`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete'`, `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, `data-action="kbPlaudConnect"`, `data-action="kbPlaudDisconnect"`, diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 718fbbf15f..0c0c2c8f71 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -618,3 +618,9 @@ project.workflow_bindings[].name # set through the per-repo workflow select on S project.workflow_bindings[].repo # see project.workflow_bindings[].name project.workflow_bindings[].version # see project.workflow_bindings[].name workflow.label_gc # operator safety knob for model-label garbage collection on rebind; yaml-only, no Settings control planned (RFC #17620) +inbox.sources[] # which voice connector types feed the Inbox; defaults to every configured plaud/wisprflow connector, yaml-only (Inbox RFC) +inbox.rules[].source # auto-routing rules are edited as whole rows in Inbox → Auto-routing via POST /api/inbox/settings, not leaf-path controls (Inbox RFC) +inbox.rules[].title_prefix # see inbox.rules[].source +inbox.rules[].tag # see inbox.rules[].source +inbox.rules[].destination # see inbox.rules[].source +inbox.rules[].repo # see inbox.rules[].source diff --git a/src/pkg/dashboard/static-src/css/inbox.css b/src/pkg/dashboard/static-src/css/inbox.css new file mode 100644 index 0000000000..c81646cfae --- /dev/null +++ b/src/pkg/dashboard/static-src/css/inbox.css @@ -0,0 +1,17 @@ + /* Inbox (voice recordings / dictations triage) */ + .inbox-toolbar { flex-wrap: wrap; gap: var(--sp-3); margin-bottom: var(--sp-4); } + .inbox-sources { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .inbox-toggle { display: inline-flex; align-items: center; gap: var(--sp-2); } + .inbox-list { max-height: none; } + .inbox-item { cursor: default; } + .inbox-check { display: flex; align-items: flex-start; padding-top: var(--sp-1); } + .inbox-icon { flex-shrink: 0; font-size: var(--fs-lg); line-height: 1; padding-top: var(--sp-1); } + .inbox-pills { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); margin-top: var(--sp-2); } + .inbox-link { font-size: var(--fs-xs); } + .inbox-field { display: block; margin-top: var(--sp-4); } + .inbox-settings-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--sp-4); } + .inbox-rule { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.4fr) auto; gap: var(--sp-2); align-items: center; margin-bottom: var(--sp-2); } + .kb-plaud-paste textarea { width: 100%; margin-top: var(--sp-2); font-family: var(--font-mono); font-size: var(--fs-xs); } + @media (max-width: 720px) { + .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } + } diff --git a/src/pkg/dashboard/static-src/html/inbox.html b/src/pkg/dashboard/static-src/html/inbox.html new file mode 100644 index 0000000000..8b462ea673 --- /dev/null +++ b/src/pkg/dashboard/static-src/html/inbox.html @@ -0,0 +1,6 @@ + <div class="mt-6 mb-8" id="inbox-section" data-dashboard-section="inbox-section"> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="inbox-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Inbox section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📥 Inbox</h2> + <div class="section-body"> + <div id="inbox-panel" class="inbox-panel"></div> + </div> + </div> diff --git a/src/pkg/dashboard/static-src/html/sidebar.html b/src/pkg/dashboard/static-src/html/sidebar.html index 434c15aea4..9ea8dbb310 100644 --- a/src/pkg/dashboard/static-src/html/sidebar.html +++ b/src/pkg/dashboard/static-src/html/sidebar.html @@ -42,6 +42,7 @@ <a class="oc-nav-item" data-section="acmm-eval-section" data-action="ocNavigate" data-arg0="acmm-eval-section"><span class="oc-nav-emoji">🎯</span><span class="oc-nav-text">ACMM Eval</span></a> <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> + <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> </div> <div class="oc-nav-group"> diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js new file mode 100644 index 0000000000..4f38662937 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -0,0 +1,296 @@ + // ── Inbox: source-agnostic triage of voice recordings and dictations ── + // Items come from knowledge connectors (plaud, wisprflow, …); each one is + // already a Knowledge fact. Routing them to spek (RFC + child issues) or + // dibs happens inside the hive via POST /api/inbox/route. + const INBOX_POLL_MS = 60000; + const INBOX_BUSY_POLL_MS = 5000; + const INBOX_SOURCE_ICONS = { plaud: '🎙️', wisprflow: '🗣️', 'file-drop': '📎', webhook: '🔗' }; + const INBOX_DEST_LABELS = { spek_rfc: 'RFC', dibs: 'dibs' }; + let _inboxCache = null; + let _inboxSelected = new Set(); + let _inboxShowProcessed = false; + let _inboxBusyTimer = null; + + function inboxSourceIcon(src) { return INBOX_SOURCE_ICONS[src] || '📥'; } + function inboxItemKey(it) { return it.source + '/' + it.id; } + function inboxFmtDuration(ms) { + const s = Math.round((Number(ms) || 0) / 1000); + if (s <= 0) return ''; + const m = Math.floor(s / 60); + return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : m + 'm ' + (s % 60) + 's'; + } + function inboxRouteBusy(items) { + return (items || []).some(it => Object.values(it.routes || {}).some(r => r && (r.status === 'queued' || r.status === 'running'))); + } + + async function fetchInbox() { + try { + const r = await fetch('/api/inbox'); + if (!r.ok) return; + _inboxCache = await r.json(); + renderInbox(); + clearTimeout(_inboxBusyTimer); + if (inboxRouteBusy(_inboxCache.items)) _inboxBusyTimer = setTimeout(fetchInbox, INBOX_BUSY_POLL_MS); + } catch (e) { /* transient — leave last render in place */ } + } + dashboardScheduleWork('inbox', fetchInbox).catch(console.error); + dashboardSetInterval('inbox', fetchInbox, INBOX_POLL_MS); + + function inboxStatePill(dest, st) { + const label = INBOX_DEST_LABELS[dest] || dest; + if (!st) return ''; + const status = st.status === 'filed' || st.status === 'delegated' ? 'ok' : (st.status === 'failed' ? 'error' : 'syncing'); + const text = st.status === 'filed' ? label + ' filed' : st.status === 'delegated' ? label + ' handed to agent' : label + ' ' + st.status; + const title = st.error ? escapeHtml(st.error) : (st.repo ? escapeHtml(st.repo) : ''); + let html = '<span class="kc-pill" data-status="' + status + '" title="' + title + '">' + escapeHtml(text) + '</span>'; + if (st.url) html += ' <a class="inbox-link" href="' + escapeHtml(st.url) + '" target="_blank" rel="noopener">open ↗</a>'; + if (Array.isArray(st.children) && st.children.length) html += ' <span class="muted-xs">' + st.children.length + ' child issue' + (st.children.length === 1 ? '' : 's') + '</span>'; + return html; + } + + function inboxItemHtml(it) { + const key = inboxItemKey(it); + const checked = _inboxSelected.has(key) ? ' checked' : ''; + const when = it.recorded_at && !String(it.recorded_at).startsWith('0001') ? relativeTimeShort(it.recorded_at) : ''; + const dur = inboxFmtDuration(it.duration_ms); + const meta = [escapeHtml(it.source), when, dur].filter(Boolean).join(' · '); + const routes = it.routes || {}; + let pills = ''; + for (const d of Object.keys(routes)) pills += inboxStatePill(d, routes[d]) + ' '; + if (it.dismissed) pills += '<span class="kc-pill" title="Kept as a Knowledge fact only">knowledge only</span>'; + const owner = viewerIsOwner(); + const src = escapeHtml(it.source), id = escapeHtml(it.id); + let actions = ''; + if (owner) { + const retry = (d) => routes[d] && routes[d].status === 'failed' ? 'Retry ' : '→ '; + actions += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="spek_rfc">' + retry('spek_rfc') + 'RFC…</button>'; + actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="dibs">' + retry('dibs') + 'dibs…</button>'; + if (!it.dismissed) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismiss" data-arg0="' + src + '" data-arg1="' + id + '" title="Keep it as a Knowledge fact; no RFC, no dibs idea">Knowledge only</button>'; + else actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxUndismiss" data-arg0="' + src + '" data-arg1="' + id + '">Back to inbox</button>'; + } + if (it.fact_slug) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenFact" data-arg0="' + escapeHtml(it.fact_slug) + '" data-arg1="' + escapeHtml(it.layer || '') + '" title="This item is already a Knowledge fact">📚 Fact</button>'; + return '<div class="kb-fact-item inbox-item" data-inbox-key="' + escapeHtml(key) + '">' + + (owner ? '<label class="inbox-check"><input type="checkbox" data-change-action="inboxToggleSelect" data-arg0="' + src + '" data-arg1="' + id + '"' + checked + ' aria-label="Select ' + escapeHtml(it.title) + '"></label>' : '') + + '<div class="inbox-icon" title="' + src + '">' + inboxSourceIcon(it.source) + '</div>' + + '<div class="kb-fact-body"><div class="kb-fact-title">' + escapeHtml(it.title || it.id) + '</div>' + + '<div class="muted-xs">' + meta + '</div>' + + (it.excerpt ? '<div class="kb-fact-snippet">' + escapeHtml(it.excerpt) + '</div>' : '') + + (pills ? '<div class="inbox-pills">' + pills + '</div>' : '') + + '<div class="kc-actions mt-2">' + actions + '</div></div></div>'; + } + + function renderInbox() { + refreshSectionCardShell('inbox-section'); + const panel = document.getElementById('inbox-panel'); + const badge = document.getElementById('inbox-sidebar-badge'); + const data = _inboxCache; + if (!panel) return; + if (!data) { panel.innerHTML = '<div class="muted-note">Loading inbox…</div>'; return; } + const items = data.items || []; + const unprocessed = Number(data.unprocessed) || 0; + if (badge) setSidebarCountBadge(badge, unprocessed, unprocessed + ' unprocessed'); + setSectionSummary('inbox-section', unprocessed + ' to triage · ' + items.length + ' total'); + const owner = viewerIsOwner(); + const sources = (data.sources || []).map(s => { + const status = s.connected ? 'ok' : (s.configured ? 'error' : ''); + const label = inboxSourceIcon(s.type) + ' ' + escapeHtml(s.type) + (s.connected ? (s.account ? ': ' + escapeHtml(s.account) : ': connected') : s.configured ? ': not connected' : ': not configured'); + return '<span class="kc-pill" data-status="' + status + '">' + label + '</span>'; + }).join(' '); + const settings = data.settings || {}; + const rules = Array.isArray(settings.rules) ? settings.rules.length : 0; + let html = '<div class="row-between inbox-toolbar">' + + '<div class="inbox-sources">' + (sources || '<span class="muted-xs">No voice sources configured.</span>') + + ' <button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenIntegrations">Connect sources</button></div>' + + '<div class="kc-actions">' + + '<label class="muted-xs inbox-toggle"><input type="checkbox" data-change-action="inboxToggleProcessed"' + (_inboxShowProcessed ? ' checked' : '') + '> show processed</label>' + + (owner ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenSettings" title="Auto-routing rules, default RFC repository, dibs URL">⚙ Auto-routing (' + rules + ')</button>' : '') + + '</div></div>'; + if (data.error) html += '<div class="kc-error muted-note">' + escapeHtml(data.error) + '</div>'; + if (!settings.spek_ready && owner) html += '<p class="muted-xs">RFC filing needs a <code>spektacular</code> binary on the hive host (<code>inbox.spek.binary</code>) or at least one enabled agent to hand the job to (<code>inbox.spek.agent</code> overrides the default).</p>'; + const shown = items.filter(it => _inboxShowProcessed || !it.processed); + if (owner && shown.length) { + const sel = _inboxSelected.size; + html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; + } + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; + panel.innerHTML = html; + } + + function inboxToggleProcessed() { _inboxShowProcessed = !_inboxShowProcessed; renderInbox(); } + function inboxToggleSelect(src, id) { + const key = src + '/' + id; + if (_inboxSelected.has(key)) _inboxSelected.delete(key); else _inboxSelected.add(key); + renderInbox(); + } + function inboxSelectAll() { + const shown = ((_inboxCache || {}).items || []).filter(it => _inboxShowProcessed || !it.processed); + if (_inboxSelected.size === shown.length) _inboxSelected.clear(); + else _inboxSelected = new Set(shown.map(inboxItemKey)); + renderInbox(); + } + function inboxFind(src, id) { return ((_inboxCache || {}).items || []).find(it => it.source === src && it.id === id); } + function inboxOpenFact(slug, layer) { + ocNavigate('knowledge-section'); + kbReadFact(slug, layer || undefined); + } + + async function inboxPostDismiss(items, undo) { + const r = await fetch('/api/inbox/dismiss', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ items, undo: !!undo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + return data; + } + async function inboxDismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], false); showToast('Kept as a Knowledge fact only', 'success'); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + async function inboxUndismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], true); await fetchInbox(); } + catch (e) { showToast('Undo failed: ' + e.message, 'error'); } + } + async function inboxDismissSelected() { + const items = Array.from(_inboxSelected).map(k => { const i = k.indexOf('/'); return { source: k.slice(0, i), item_id: k.slice(i + 1) }; }); + if (!items.length) return; + if (!(await hiveConfirm('Mark ' + items.length + ' item' + (items.length === 1 ? '' : 's') + ' as Knowledge only? They stay searchable as facts; no RFC or dibs idea is created.', 'Knowledge only'))) return; + try { await inboxPostDismiss(items, false); _inboxSelected.clear(); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + + // Route dialog: destination-specific confirm with a repo picker for RFCs. + function inboxOpenRoute(src, id, dest) { + const it = inboxFind(src, id); + if (!it) return; + const settings = (_inboxCache || {}).settings || {}; + const repos = (_inboxCache || {}).repos || []; + const def = settings.spek_repo || repos[0] || ''; + let body = '<p class="muted-note m-0"><strong>' + escapeHtml(it.title || it.id) + '</strong></p>'; + if (dest === 'spek_rfc') { + const opts = repos.map(r => '<option value="' + escapeHtml(r) + '"' + (r === def ? ' selected' : '') + '>' + escapeHtml(r) + '</option>').join(''); + body += '<p class="muted-note">A hive-run <code>spektacular voice</code> turns the transcript into a spec, then files one RFC issue plus child issues on the chosen repository. The children are picked up by the hive like any other issue.</p>' + + '<label class="muted-note inbox-field">Repository<select class="field-input field-input-block mt-2" id="inbox-route-repo">' + opts + '</select></label>' + + (repos.length ? '' : '<p class="kc-error muted-xs">No repositories configured on this hive.</p>'); + } else { + body += '<p class="muted-note">Posts the title, summary and transcript to dibs as a private draft idea' + (settings.dibs_url ? ' at <code>' + escapeHtml(settings.dibs_url) + '</code>' : '') + '. Provenance links back to <code>' + escapeHtml(src) + '://recordings/' + escapeHtml(id) + '</code>.</p>' + + (!settings.dibs_url ? '<p class="kc-error muted-xs">dibs is not configured — set the URL under ⚙ Auto-routing and <code>inbox.dibs.token_env</code> in the hive config.</p>' : !settings.dibs_token_set ? '<p class="kc-error muted-xs">dibs token missing — set the environment variable named by <code>inbox.dibs.token_env</code>.</p>' : ''); + } + body += '<div class="hive-dialog-error" role="alert" id="inbox-route-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxConfirmRoute" data-arg0="' + escapeHtml(src) + '" data-arg1="' + escapeHtml(id) + '" data-arg2="' + escapeHtml(dest) + '">' + (dest === 'spek_rfc' ? 'File RFC' : 'Send to dibs') + '</button></div>'; + kbShowModal(dest === 'spek_rfc' ? '→ RFC (spektacular)' : '→ dibs', body, { width: '520px' }); + } + async function inboxConfirmRoute(src, id, dest) { + const repoEl = document.getElementById('inbox-route-repo'); + const repo = repoEl ? repoEl.value : ''; + const errEl = document.getElementById('inbox-route-error'); + try { + const r = await fetch('/api/inbox/route', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ source: src, item_id: id, destination: dest, repo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast((INBOX_DEST_LABELS[dest] || dest) + ' queued — the pill updates when it finishes.', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Route failed: ' + e.message, 'error'); + } + } + + // Settings drawer: default RFC repo, dibs URL, and ordered auto-routing rules. + let _inboxDraftRules = []; + function inboxRuleRowHtml(rule, i) { + const sources = ((_inboxCache || {}).settings || {}).sources || []; + const srcOpts = ['<option value=""' + (!rule.source ? ' selected' : '') + '>any source</option>'].concat(sources.map(s => '<option value="' + escapeHtml(s) + '"' + (rule.source === s ? ' selected' : '') + '>' + escapeHtml(s) + '</option>')).join(''); + const destOpts = [['spek_rfc', '→ RFC'], ['dibs', '→ dibs'], ['dismiss', 'Knowledge only']].map(([v, l]) => '<option value="' + v + '"' + (rule.destination === v ? ' selected' : '') + '>' + l + '</option>').join(''); + return '<div class="inbox-rule" data-rule-index="' + i + '">' + + '<select class="field-input" data-rule-field="source" aria-label="Source">' + srcOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="title_prefix" placeholder="title prefix (RFC:)" value="' + escapeHtml(rule.title_prefix || '') + '" aria-label="Title prefix">' + + '<input class="kb-search-box" data-rule-field="tag" placeholder="or tag" value="' + escapeHtml(rule.tag || '') + '" aria-label="Tag">' + + '<select class="field-input" data-rule-field="destination" aria-label="Destination">' + destOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; + } + function inboxReadRules() { + return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; + return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; + }); + } + function inboxRenderRules() { + const host = document.getElementById('inbox-settings-rules'); + if (!host) return; + host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + '<div class="inbox-settings-grid">' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + + '<div id="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + inboxRenderRules(); + } + function inboxAddRule() { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); + inboxRenderRules(); + } + function inboxRemoveRule(i) { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.splice(Number(i), 1); + inboxRenderRules(); + } + async function inboxSaveSettings() { + const errEl = document.getElementById('inbox-settings-error'); + const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + try { + const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast('Inbox settings saved', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Save failed: ' + e.message, 'error'); + } + } + + // Hosted-hive Plaud sign-in completion: Plaud's OAuth client only redirects + // to localhost:8199, so the operator pastes either the redirect URL the + // browser landed on or the tokens-mcp.json contents; the hive finishes + // the login in-pod. + async function kbPlaudPasteComplete(name) { + const el = document.getElementById('kb-plaud-paste-' + name); + const raw = el ? el.value.trim() : ''; + if (!raw) { showToast('Paste the redirect URL or the tokens JSON first.', 'error'); return; } + const body = raw.startsWith('{') ? { tokens: raw } : { redirect_url: raw }; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast(data.connected ? 'Plaud connected' + (data.account ? ' as ' + data.account : '') : 'Plaud sign-in submitted; click Check.', data.connected ? 'success' : 'info'); + fetchInbox(); + } catch (e) { + showToast('Plaud sign-in failed: ' + e.message, 'error'); + } + } + function kbPlaudPasteHtml(name) { + const n = escapeHtml(name); + const a = _kbPlaudAuth[name] || {}; + const openInbox = '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="ocNavigate" data-arg0="inbox-section">📥 Open Inbox</button>'; + if (a.connected) return '<div class="kc-actions mt-2">' + openInbox + '<span class="muted-xs">Recordings are triaged in the Inbox, not here.</span></div>'; + return '<details class="kb-plaud-paste mt-2"><summary class="muted-xs">Hive runs on a server? Finish sign-in by pasting</summary>' + + '<p class="muted-xs">Plaud only redirects to <code>localhost:8199</code>. Open the authorization link, approve, then paste the <code>localhost:8199/auth/callback?…</code> URL your browser lands on (within 10 minutes). Alternatively paste the contents of <code>~/.plaud/tokens-mcp.json</code> from a laptop where <code>plaud-mcp</code> is signed in.</p>' + + '<textarea class="kb-search-box" id="kb-plaud-paste-' + n + '" rows="3" placeholder="http://localhost:8199/auth/callback?code=… or {"access_token": …}"></textarea>' + + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudPasteComplete" data-arg0="' + n + '">Submit</button>' + openInbox + '</div></details>'; + } diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 81daf98c72..10fdd87fd5 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2433,7 +2433,7 @@ html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { @@ -2572,7 +2572,7 @@ } let text; if (a.error) text = 'Plaud: ' + a.error; - else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a> (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a>' : ' (URL pending…)'); else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as <strong>' + escapeHtml(a.account) + '</strong>' : ''); else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; diff --git a/src/pkg/dashboard/static-src/js/layout.js b/src/pkg/dashboard/static-src/js/layout.js index 3819b9027b..4e6e3f3a67 100644 --- a/src/pkg/dashboard/static-src/js/layout.js +++ b/src/pkg/dashboard/static-src/js/layout.js @@ -4,7 +4,7 @@ var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; diff --git a/src/pkg/dashboard/static-src/js/refresh.js b/src/pkg/dashboard/static-src/js/refresh.js index 21511384cf..8e4e569066 100644 --- a/src/pkg/dashboard/static-src/js/refresh.js +++ b/src/pkg/dashboard/static-src/js/refresh.js @@ -562,13 +562,14 @@ ocUpdateSidebarAgents(); dashboardRenderSection('debug', ['debug-section'], () => renderDebugSection(), data); dashboardRenderSection('faq', ['faq-section'], () => renderFAQ(), data); + dashboardRenderSection('inbox', ['inbox-section'], () => renderInbox(), data); initializeDashboardSectionCards(); // Restore persisted collapsed state for all collapsible sections const COLLAPSIBLE_SECTION_IDS = [ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', - 'knowledge-section', 'contributors-section', 'debug-section', 'logs-section', + 'knowledge-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; COLLAPSIBLE_SECTION_IDS.forEach(applySectionCollapse); diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index a289e2d9aa..35e32dd311 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -9,7 +9,7 @@ // marker key records whether the browser predates the defaults ('legacy', // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ - 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', + 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section' ]); @@ -121,6 +121,7 @@ 'nous-section': { title: '🧪 Experiment Lab (nous)', summary: '—', settingsHtml: sectionSettingsGear('Experiment Lab', 'openNousConfig', 'Experiment Lab Configuration') }, 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, + 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, 'debug-section': { title: '🩺 Diagnostics', summary: 'health' }, 'logs-section': { title: '📋 Agent Logs', summary: 'logs' }, @@ -159,6 +160,7 @@ 'nous-section': { heading: 'Experiment Lab', help: 'An experimental planner that tries out code changes in your repositories and records what it learned.' }, 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, + 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, 'contributors-section': { heading: 'Contributors', help: 'The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them.' }, 'debug-section': { heading: 'Diagnostics', help: 'Health checks for Hive itself, such as its connections, credentials and safety switches, to help you find out why something is not working.' }, 'logs-section': { heading: 'Agent Logs', help: 'The raw text output of each worker, for following what it is doing line by line.' }, diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index 45731eba3a..d6c8e81100 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -29,6 +29,7 @@ css/workflows-marketplace.css css/chat.css css/nous.css css/knowledge.css +css/inbox.css css/sections-collapse.css css/design-system.css css/responsive.css @@ -56,6 +57,7 @@ html/repos.html html/acmm-approvals-audit.html html/review-queue.html html/nous-inception-knowledge-contributors.html +html/inbox.html html/debug-logs.html html/agents.html html/tracker.html @@ -119,6 +121,7 @@ js/evidence.js js/approvals.js js/nous.js js/knowledge.js +js/inbox.js js/vault-audit.js js/review-queue.js js/contributors.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index e75ead995d..90af378a72 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5920,6 +5920,23 @@ .field-error { color: var(--red); font-size: var(--fs-xs); margin-top: var(--sp-1); } @media (max-width: 768px) { .kb-layout { grid-template-columns: 1fr; } } + /* Inbox (voice recordings / dictations triage) */ + .inbox-toolbar { flex-wrap: wrap; gap: var(--sp-3); margin-bottom: var(--sp-4); } + .inbox-sources { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .inbox-toggle { display: inline-flex; align-items: center; gap: var(--sp-2); } + .inbox-list { max-height: none; } + .inbox-item { cursor: default; } + .inbox-check { display: flex; align-items: flex-start; padding-top: var(--sp-1); } + .inbox-icon { flex-shrink: 0; font-size: var(--fs-lg); line-height: 1; padding-top: var(--sp-1); } + .inbox-pills { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); margin-top: var(--sp-2); } + .inbox-link { font-size: var(--fs-xs); } + .inbox-field { display: block; margin-top: var(--sp-4); } + .inbox-settings-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--sp-4); } + .inbox-rule { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.4fr) auto; gap: var(--sp-2); align-items: center; margin-bottom: var(--sp-2); } + .kb-plaud-paste textarea { width: 100%; margin-top: var(--sp-2); font-family: var(--font-mono); font-size: var(--fs-xs); } + @media (max-width: 720px) { + .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } + } /* Collapsible advisory sections (#1546) */ .collapse-chevron { display: inline-block; @@ -8560,6 +8577,7 @@ <h2 class="config-title">🐝 Getting Started<span id="welcome-progress" class=" <a class="oc-nav-item" data-section="acmm-eval-section" data-action="ocNavigate" data-arg0="acmm-eval-section"><span class="oc-nav-emoji">🎯</span><span class="oc-nav-text">ACMM Eval</span></a> <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> + <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> </div> <div class="oc-nav-group"> @@ -8998,6 +9016,12 @@ <h2 class="section-label m-0">👥 Contributors <span id="contributors-status-ba </div> + <div class="mt-6 mb-8" id="inbox-section" data-dashboard-section="inbox-section"> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="inbox-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Inbox section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📥 Inbox</h2> + <div class="section-body"> + <div id="inbox-panel" class="inbox-panel"></div> + </div> + </div> <div class="mt-6 mb-8" id="debug-section" data-dashboard-section="debug-section"> <h2 class="section-label section-header-toggle mb-5" data-action="toggleSection" data-arg0="debug-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Diagnostics section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 🩺 Diagnostics</h2> <div class="section-body"> @@ -10351,7 +10375,7 @@ <h3 class="faq-group-title">Where to get help</h3> // marker key records whether the browser predates the defaults ('legacy', // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ - 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', + 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section' ]); @@ -10463,6 +10487,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'nous-section': { title: '🧪 Experiment Lab (nous)', summary: '—', settingsHtml: sectionSettingsGear('Experiment Lab', 'openNousConfig', 'Experiment Lab Configuration') }, 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, + 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, 'debug-section': { title: '🩺 Diagnostics', summary: 'health' }, 'logs-section': { title: '📋 Agent Logs', summary: 'logs' }, @@ -10501,6 +10526,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'nous-section': { heading: 'Experiment Lab', help: 'An experimental planner that tries out code changes in your repositories and records what it learned.' }, 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, + 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, 'contributors-section': { heading: 'Contributors', help: 'The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them.' }, 'debug-section': { heading: 'Diagnostics', help: 'Health checks for Hive itself, such as its connections, credentials and safety switches, to help you find out why something is not working.' }, 'logs-section': { heading: 'Agent Logs', help: 'The raw text output of each worker, for following what it is doing line by line.' }, @@ -11925,7 +11951,7 @@ <h3 class="faq-group-title">Where to get help</h3> var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; @@ -45953,7 +45979,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { @@ -46092,7 +46118,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> } let text; if (a.error) text = 'Plaud: ' + a.error; - else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a> (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a>' : ' (URL pending…)'); else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as <strong>' + escapeHtml(a.account) + '</strong>' : ''); else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; @@ -46168,6 +46194,302 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> return '<div class="muted-xs" data-kb-wisprflow-source>Wispr Flow: ' + (path ? '<code>' + escapeHtml(path) + '</code>' : 'default app location on the hive host') + ' · ' + group + '</div>'; } function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + // ── Inbox: source-agnostic triage of voice recordings and dictations ── + // Items come from knowledge connectors (plaud, wisprflow, …); each one is + // already a Knowledge fact. Routing them to spek (RFC + child issues) or + // dibs happens inside the hive via POST /api/inbox/route. + const INBOX_POLL_MS = 60000; + const INBOX_BUSY_POLL_MS = 5000; + const INBOX_SOURCE_ICONS = { plaud: '🎙️', wisprflow: '🗣️', 'file-drop': '📎', webhook: '🔗' }; + const INBOX_DEST_LABELS = { spek_rfc: 'RFC', dibs: 'dibs' }; + let _inboxCache = null; + let _inboxSelected = new Set(); + let _inboxShowProcessed = false; + let _inboxBusyTimer = null; + + function inboxSourceIcon(src) { return INBOX_SOURCE_ICONS[src] || '📥'; } + function inboxItemKey(it) { return it.source + '/' + it.id; } + function inboxFmtDuration(ms) { + const s = Math.round((Number(ms) || 0) / 1000); + if (s <= 0) return ''; + const m = Math.floor(s / 60); + return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : m + 'm ' + (s % 60) + 's'; + } + function inboxRouteBusy(items) { + return (items || []).some(it => Object.values(it.routes || {}).some(r => r && (r.status === 'queued' || r.status === 'running'))); + } + + async function fetchInbox() { + try { + const r = await fetch('/api/inbox'); + if (!r.ok) return; + _inboxCache = await r.json(); + renderInbox(); + clearTimeout(_inboxBusyTimer); + if (inboxRouteBusy(_inboxCache.items)) _inboxBusyTimer = setTimeout(fetchInbox, INBOX_BUSY_POLL_MS); + } catch (e) { /* transient — leave last render in place */ } + } + dashboardScheduleWork('inbox', fetchInbox).catch(console.error); + dashboardSetInterval('inbox', fetchInbox, INBOX_POLL_MS); + + function inboxStatePill(dest, st) { + const label = INBOX_DEST_LABELS[dest] || dest; + if (!st) return ''; + const status = st.status === 'filed' || st.status === 'delegated' ? 'ok' : (st.status === 'failed' ? 'error' : 'syncing'); + const text = st.status === 'filed' ? label + ' filed' : st.status === 'delegated' ? label + ' handed to agent' : label + ' ' + st.status; + const title = st.error ? escapeHtml(st.error) : (st.repo ? escapeHtml(st.repo) : ''); + let html = '<span class="kc-pill" data-status="' + status + '" title="' + title + '">' + escapeHtml(text) + '</span>'; + if (st.url) html += ' <a class="inbox-link" href="' + escapeHtml(st.url) + '" target="_blank" rel="noopener">open ↗</a>'; + if (Array.isArray(st.children) && st.children.length) html += ' <span class="muted-xs">' + st.children.length + ' child issue' + (st.children.length === 1 ? '' : 's') + '</span>'; + return html; + } + + function inboxItemHtml(it) { + const key = inboxItemKey(it); + const checked = _inboxSelected.has(key) ? ' checked' : ''; + const when = it.recorded_at && !String(it.recorded_at).startsWith('0001') ? relativeTimeShort(it.recorded_at) : ''; + const dur = inboxFmtDuration(it.duration_ms); + const meta = [escapeHtml(it.source), when, dur].filter(Boolean).join(' · '); + const routes = it.routes || {}; + let pills = ''; + for (const d of Object.keys(routes)) pills += inboxStatePill(d, routes[d]) + ' '; + if (it.dismissed) pills += '<span class="kc-pill" title="Kept as a Knowledge fact only">knowledge only</span>'; + const owner = viewerIsOwner(); + const src = escapeHtml(it.source), id = escapeHtml(it.id); + let actions = ''; + if (owner) { + const retry = (d) => routes[d] && routes[d].status === 'failed' ? 'Retry ' : '→ '; + actions += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="spek_rfc">' + retry('spek_rfc') + 'RFC…</button>'; + actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="dibs">' + retry('dibs') + 'dibs…</button>'; + if (!it.dismissed) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismiss" data-arg0="' + src + '" data-arg1="' + id + '" title="Keep it as a Knowledge fact; no RFC, no dibs idea">Knowledge only</button>'; + else actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxUndismiss" data-arg0="' + src + '" data-arg1="' + id + '">Back to inbox</button>'; + } + if (it.fact_slug) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenFact" data-arg0="' + escapeHtml(it.fact_slug) + '" data-arg1="' + escapeHtml(it.layer || '') + '" title="This item is already a Knowledge fact">📚 Fact</button>'; + return '<div class="kb-fact-item inbox-item" data-inbox-key="' + escapeHtml(key) + '">' + + (owner ? '<label class="inbox-check"><input type="checkbox" data-change-action="inboxToggleSelect" data-arg0="' + src + '" data-arg1="' + id + '"' + checked + ' aria-label="Select ' + escapeHtml(it.title) + '"></label>' : '') + + '<div class="inbox-icon" title="' + src + '">' + inboxSourceIcon(it.source) + '</div>' + + '<div class="kb-fact-body"><div class="kb-fact-title">' + escapeHtml(it.title || it.id) + '</div>' + + '<div class="muted-xs">' + meta + '</div>' + + (it.excerpt ? '<div class="kb-fact-snippet">' + escapeHtml(it.excerpt) + '</div>' : '') + + (pills ? '<div class="inbox-pills">' + pills + '</div>' : '') + + '<div class="kc-actions mt-2">' + actions + '</div></div></div>'; + } + + function renderInbox() { + refreshSectionCardShell('inbox-section'); + const panel = document.getElementById('inbox-panel'); + const badge = document.getElementById('inbox-sidebar-badge'); + const data = _inboxCache; + if (!panel) return; + if (!data) { panel.innerHTML = '<div class="muted-note">Loading inbox…</div>'; return; } + const items = data.items || []; + const unprocessed = Number(data.unprocessed) || 0; + if (badge) setSidebarCountBadge(badge, unprocessed, unprocessed + ' unprocessed'); + setSectionSummary('inbox-section', unprocessed + ' to triage · ' + items.length + ' total'); + const owner = viewerIsOwner(); + const sources = (data.sources || []).map(s => { + const status = s.connected ? 'ok' : (s.configured ? 'error' : ''); + const label = inboxSourceIcon(s.type) + ' ' + escapeHtml(s.type) + (s.connected ? (s.account ? ': ' + escapeHtml(s.account) : ': connected') : s.configured ? ': not connected' : ': not configured'); + return '<span class="kc-pill" data-status="' + status + '">' + label + '</span>'; + }).join(' '); + const settings = data.settings || {}; + const rules = Array.isArray(settings.rules) ? settings.rules.length : 0; + let html = '<div class="row-between inbox-toolbar">' + + '<div class="inbox-sources">' + (sources || '<span class="muted-xs">No voice sources configured.</span>') + + ' <button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenIntegrations">Connect sources</button></div>' + + '<div class="kc-actions">' + + '<label class="muted-xs inbox-toggle"><input type="checkbox" data-change-action="inboxToggleProcessed"' + (_inboxShowProcessed ? ' checked' : '') + '> show processed</label>' + + (owner ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenSettings" title="Auto-routing rules, default RFC repository, dibs URL">⚙ Auto-routing (' + rules + ')</button>' : '') + + '</div></div>'; + if (data.error) html += '<div class="kc-error muted-note">' + escapeHtml(data.error) + '</div>'; + if (!settings.spek_ready && owner) html += '<p class="muted-xs">RFC filing needs a <code>spektacular</code> binary on the hive host (<code>inbox.spek.binary</code>) or at least one enabled agent to hand the job to (<code>inbox.spek.agent</code> overrides the default).</p>'; + const shown = items.filter(it => _inboxShowProcessed || !it.processed); + if (owner && shown.length) { + const sel = _inboxSelected.size; + html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; + } + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; + panel.innerHTML = html; + } + + function inboxToggleProcessed() { _inboxShowProcessed = !_inboxShowProcessed; renderInbox(); } + function inboxToggleSelect(src, id) { + const key = src + '/' + id; + if (_inboxSelected.has(key)) _inboxSelected.delete(key); else _inboxSelected.add(key); + renderInbox(); + } + function inboxSelectAll() { + const shown = ((_inboxCache || {}).items || []).filter(it => _inboxShowProcessed || !it.processed); + if (_inboxSelected.size === shown.length) _inboxSelected.clear(); + else _inboxSelected = new Set(shown.map(inboxItemKey)); + renderInbox(); + } + function inboxFind(src, id) { return ((_inboxCache || {}).items || []).find(it => it.source === src && it.id === id); } + function inboxOpenFact(slug, layer) { + ocNavigate('knowledge-section'); + kbReadFact(slug, layer || undefined); + } + + async function inboxPostDismiss(items, undo) { + const r = await fetch('/api/inbox/dismiss', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ items, undo: !!undo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + return data; + } + async function inboxDismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], false); showToast('Kept as a Knowledge fact only', 'success'); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + async function inboxUndismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], true); await fetchInbox(); } + catch (e) { showToast('Undo failed: ' + e.message, 'error'); } + } + async function inboxDismissSelected() { + const items = Array.from(_inboxSelected).map(k => { const i = k.indexOf('/'); return { source: k.slice(0, i), item_id: k.slice(i + 1) }; }); + if (!items.length) return; + if (!(await hiveConfirm('Mark ' + items.length + ' item' + (items.length === 1 ? '' : 's') + ' as Knowledge only? They stay searchable as facts; no RFC or dibs idea is created.', 'Knowledge only'))) return; + try { await inboxPostDismiss(items, false); _inboxSelected.clear(); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + + // Route dialog: destination-specific confirm with a repo picker for RFCs. + function inboxOpenRoute(src, id, dest) { + const it = inboxFind(src, id); + if (!it) return; + const settings = (_inboxCache || {}).settings || {}; + const repos = (_inboxCache || {}).repos || []; + const def = settings.spek_repo || repos[0] || ''; + let body = '<p class="muted-note m-0"><strong>' + escapeHtml(it.title || it.id) + '</strong></p>'; + if (dest === 'spek_rfc') { + const opts = repos.map(r => '<option value="' + escapeHtml(r) + '"' + (r === def ? ' selected' : '') + '>' + escapeHtml(r) + '</option>').join(''); + body += '<p class="muted-note">A hive-run <code>spektacular voice</code> turns the transcript into a spec, then files one RFC issue plus child issues on the chosen repository. The children are picked up by the hive like any other issue.</p>' + + '<label class="muted-note inbox-field">Repository<select class="field-input field-input-block mt-2" id="inbox-route-repo">' + opts + '</select></label>' + + (repos.length ? '' : '<p class="kc-error muted-xs">No repositories configured on this hive.</p>'); + } else { + body += '<p class="muted-note">Posts the title, summary and transcript to dibs as a private draft idea' + (settings.dibs_url ? ' at <code>' + escapeHtml(settings.dibs_url) + '</code>' : '') + '. Provenance links back to <code>' + escapeHtml(src) + '://recordings/' + escapeHtml(id) + '</code>.</p>' + + (!settings.dibs_url ? '<p class="kc-error muted-xs">dibs is not configured — set the URL under ⚙ Auto-routing and <code>inbox.dibs.token_env</code> in the hive config.</p>' : !settings.dibs_token_set ? '<p class="kc-error muted-xs">dibs token missing — set the environment variable named by <code>inbox.dibs.token_env</code>.</p>' : ''); + } + body += '<div class="hive-dialog-error" role="alert" id="inbox-route-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxConfirmRoute" data-arg0="' + escapeHtml(src) + '" data-arg1="' + escapeHtml(id) + '" data-arg2="' + escapeHtml(dest) + '">' + (dest === 'spek_rfc' ? 'File RFC' : 'Send to dibs') + '</button></div>'; + kbShowModal(dest === 'spek_rfc' ? '→ RFC (spektacular)' : '→ dibs', body, { width: '520px' }); + } + async function inboxConfirmRoute(src, id, dest) { + const repoEl = document.getElementById('inbox-route-repo'); + const repo = repoEl ? repoEl.value : ''; + const errEl = document.getElementById('inbox-route-error'); + try { + const r = await fetch('/api/inbox/route', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ source: src, item_id: id, destination: dest, repo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast((INBOX_DEST_LABELS[dest] || dest) + ' queued — the pill updates when it finishes.', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Route failed: ' + e.message, 'error'); + } + } + + // Settings drawer: default RFC repo, dibs URL, and ordered auto-routing rules. + let _inboxDraftRules = []; + function inboxRuleRowHtml(rule, i) { + const sources = ((_inboxCache || {}).settings || {}).sources || []; + const srcOpts = ['<option value=""' + (!rule.source ? ' selected' : '') + '>any source</option>'].concat(sources.map(s => '<option value="' + escapeHtml(s) + '"' + (rule.source === s ? ' selected' : '') + '>' + escapeHtml(s) + '</option>')).join(''); + const destOpts = [['spek_rfc', '→ RFC'], ['dibs', '→ dibs'], ['dismiss', 'Knowledge only']].map(([v, l]) => '<option value="' + v + '"' + (rule.destination === v ? ' selected' : '') + '>' + l + '</option>').join(''); + return '<div class="inbox-rule" data-rule-index="' + i + '">' + + '<select class="field-input" data-rule-field="source" aria-label="Source">' + srcOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="title_prefix" placeholder="title prefix (RFC:)" value="' + escapeHtml(rule.title_prefix || '') + '" aria-label="Title prefix">' + + '<input class="kb-search-box" data-rule-field="tag" placeholder="or tag" value="' + escapeHtml(rule.tag || '') + '" aria-label="Tag">' + + '<select class="field-input" data-rule-field="destination" aria-label="Destination">' + destOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; + } + function inboxReadRules() { + return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; + return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; + }); + } + function inboxRenderRules() { + const host = document.getElementById('inbox-settings-rules'); + if (!host) return; + host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + '<div class="inbox-settings-grid">' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + + '<div id="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + inboxRenderRules(); + } + function inboxAddRule() { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); + inboxRenderRules(); + } + function inboxRemoveRule(i) { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.splice(Number(i), 1); + inboxRenderRules(); + } + async function inboxSaveSettings() { + const errEl = document.getElementById('inbox-settings-error'); + const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + try { + const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast('Inbox settings saved', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Save failed: ' + e.message, 'error'); + } + } + + // Hosted-hive Plaud sign-in completion: Plaud's OAuth client only redirects + // to localhost:8199, so the operator pastes either the redirect URL the + // browser landed on or the tokens-mcp.json contents; the hive finishes + // the login in-pod. + async function kbPlaudPasteComplete(name) { + const el = document.getElementById('kb-plaud-paste-' + name); + const raw = el ? el.value.trim() : ''; + if (!raw) { showToast('Paste the redirect URL or the tokens JSON first.', 'error'); return; } + const body = raw.startsWith('{') ? { tokens: raw } : { redirect_url: raw }; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast(data.connected ? 'Plaud connected' + (data.account ? ' as ' + data.account : '') : 'Plaud sign-in submitted; click Check.', data.connected ? 'success' : 'info'); + fetchInbox(); + } catch (e) { + showToast('Plaud sign-in failed: ' + e.message, 'error'); + } + } + function kbPlaudPasteHtml(name) { + const n = escapeHtml(name); + const a = _kbPlaudAuth[name] || {}; + const openInbox = '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="ocNavigate" data-arg0="inbox-section">📥 Open Inbox</button>'; + if (a.connected) return '<div class="kc-actions mt-2">' + openInbox + '<span class="muted-xs">Recordings are triaged in the Inbox, not here.</span></div>'; + return '<details class="kb-plaud-paste mt-2"><summary class="muted-xs">Hive runs on a server? Finish sign-in by pasting</summary>' + + '<p class="muted-xs">Plaud only redirects to <code>localhost:8199</code>. Open the authorization link, approve, then paste the <code>localhost:8199/auth/callback?…</code> URL your browser lands on (within 10 minutes). Alternatively paste the contents of <code>~/.plaud/tokens-mcp.json</code> from a laptop where <code>plaud-mcp</code> is signed in.</p>' + + '<textarea class="kb-search-box" id="kb-plaud-paste-' + n + '" rows="3" placeholder="http://localhost:8199/auth/callback?code=… or {"access_token": …}"></textarea>' + + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudPasteComplete" data-arg0="' + n + '">Submit</button>' + openInbox + '</div></details>'; + } // ── Vault Management ── function kbOpenVaults() { @@ -48180,13 +48502,14 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> ocUpdateSidebarAgents(); dashboardRenderSection('debug', ['debug-section'], () => renderDebugSection(), data); dashboardRenderSection('faq', ['faq-section'], () => renderFAQ(), data); + dashboardRenderSection('inbox', ['inbox-section'], () => renderInbox(), data); initializeDashboardSectionCards(); // Restore persisted collapsed state for all collapsible sections const COLLAPSIBLE_SECTION_IDS = [ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', - 'knowledge-section', 'contributors-section', 'debug-section', 'logs-section', + 'knowledge-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; COLLAPSIBLE_SECTION_IDS.forEach(applySectionCollapse); diff --git a/src/pkg/dashboard/webstatic/static_index_test.go b/src/pkg/dashboard/webstatic/static_index_test.go index 79db965065..9a1a9ad129 100644 --- a/src/pkg/dashboard/webstatic/static_index_test.go +++ b/src/pkg/dashboard/webstatic/static_index_test.go @@ -420,7 +420,7 @@ func TestStaticDashboardDefaultLayoutOrder(t *testing.T) { } html := rec.Body.String() - defaultOrder := "'overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" + defaultOrder := "'overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" if !strings.Contains(html, defaultOrder) { t.Fatalf("dashboard default layout/reset order no longer matches sidebar nav; missing %q", defaultOrder) } diff --git a/src/pkg/inbox/dibs.go b/src/pkg/inbox/dibs.go new file mode 100644 index 0000000000..334c2c203b --- /dev/null +++ b/src/pkg/inbox/dibs.go @@ -0,0 +1,113 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// dibsTimeout bounds one idea submission. +const dibsTimeout = 30 * time.Second + +// dibsMaxBody caps the idea body so a long transcript does not trip dibs' +// request limit; the full text stays in Knowledge. +const dibsMaxBody = 60_000 + +// DibsDestination posts an item as a private draft idea to a dibs instance +// (`POST {url}/api/ideas`, bearer hub token). +type DibsDestination struct { + URL string + Token string + HTTP *http.Client +} + +// Name implements Destination. +func (d *DibsDestination) Name() string { return DestDibs } + +// ErrDibsUnconfigured is returned when inbox.dibs.url is empty. +var ErrDibsUnconfigured = errors.New("dibs: inbox.dibs.url is not configured") + +type dibsIdea struct { + Title string `json:"title"` + Body string `json:"body"` + Visibility string `json:"visibility"` + Status string `json:"status"` +} + +// Deliver implements Destination. +func (d *DibsDestination) Deliver(ctx context.Context, it Item, c Content, _ Request) (Result, error) { + base := strings.TrimRight(strings.TrimSpace(d.URL), "/") + if base == "" { + return Result{}, ErrDibsUnconfigured + } + if strings.TrimSpace(d.Token) == "" { + return Result{}, errors.New("dibs: no token (set inbox.dibs.token_env to a variable holding a hub bearer token)") + } + body := DibsBody(it, c) + payload, _ := json.Marshal(dibsIdea{ + Title: firstNonEmpty(c.Title, it.Title), + Body: body, + Visibility: "private", + Status: "draft", + }) + rctx, cancel := context.WithTimeout(ctx, dibsTimeout) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodPost, base+"/api/ideas", bytes.NewReader(payload)) + if err != nil { + return Result{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+d.Token) + client := d.HTTP + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return Result{}, fmt.Errorf("dibs: %w", err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return Result{}, fmt.Errorf("dibs: %s: %s", resp.Status, lastLines(string(data), 2)) + } + var idea struct { + ID string `json:"id"` + } + if err := json.Unmarshal(data, &idea); err != nil || idea.ID == "" { + return Result{}, fmt.Errorf("dibs: response has no idea id: %s", lastLines(string(data), 2)) + } + return Result{URL: base + "/api/ideas/" + idea.ID}, nil +} + +// DibsBody renders the idea body: summary, provenance, then transcript. +func DibsBody(it Item, c Content) string { + var b strings.Builder + if c.Summary != "" { + b.WriteString(c.Summary) + b.WriteString("\n\n") + } + fmt.Fprintf(&b, "_Source: %s_\n", c.Provenance) + if !it.RecordedAt.IsZero() { + fmt.Fprintf(&b, "_Recorded: %s_\n", it.RecordedAt.UTC().Format(time.RFC3339)) + } + text := c.Transcript + if text == "" && c.Summary == "" { + text = c.Markdown + } + if text != "" { + b.WriteString("\n## Transcript\n\n") + b.WriteString(text) + } + out := b.String() + if len(out) > dibsMaxBody { + out = out[:dibsMaxBody] + "\n\n_(truncated; full transcript in Knowledge)_" + } + return out +} diff --git a/src/pkg/inbox/edges_test.go b/src/pkg/inbox/edges_test.go new file mode 100644 index 0000000000..52ea44841a --- /dev/null +++ b/src/pkg/inbox/edges_test.go @@ -0,0 +1,276 @@ +package inbox + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestServiceSettingsFindAndConfigUpdate(t *testing.T) { + src := &fakeSource{items: []Item{{Source: "plaud", ID: "rec-1", Title: "one"}}} + spek := &recordingDest{name: DestSpekRFC} + dibs := &recordingDest{name: DestDibs} + svc := newTestService(t, config.InboxConfig{}, src, spek, dibs) + if spek.Name() != DestSpekRFC || dibs.Name() != DestDibs { + t.Fatal("destination names") + } + + st := svc.Settings() + if st.Rules == nil || len(st.Rules) != 0 || st.SpekReady || st.DibsTokenSet { + t.Fatalf("default settings = %+v", st) + } + svc.SetReadiness(func() bool { return true }, func() bool { return true }) + svc.UpdateConfig(config.InboxConfig{ + Rules: []config.InboxRule{{TitlePrefix: "RFC:", Destination: DestSpekRFC}}, + Spek: config.InboxSpek{Repo: "o/r"}, + Dibs: config.InboxDibs{URL: "https://dibs.example"}, + }) + st = svc.Settings() + if !st.SpekReady || !st.DibsTokenSet || st.SpekRepo != "o/r" || st.DibsURL != "https://dibs.example" || len(st.Rules) != 1 { + t.Fatalf("settings after update = %+v", st) + } + + it, ok, err := svc.Find(context.Background(), "plaud", "rec-1") + if err != nil || !ok || it.Title != "one" { + t.Fatalf("find = %+v %v %v", it, ok, err) + } + if _, ok, _ := svc.Find(context.Background(), "plaud", "missing"); ok { + t.Fatal("found a missing item") + } + svc.Wait() +} + +func TestServiceNowAndSortItems(t *testing.T) { + fixed := time.Date(2026, 3, 4, 5, 6, 7, 0, time.FixedZone("x", 3600)) + svc := newTestService(t, config.InboxConfig{}, &fakeSource{}, &recordingDest{name: DestSpekRFC}) + if !svc.now().Before(time.Now().Add(time.Second)) { + t.Fatal("now() without override should be wall clock") + } + svc.Now = func() time.Time { return fixed } + if got := svc.now(); got != fixed.UTC() { + t.Fatalf("now = %v", got) + } + items := []Item{ + {ID: "a", RecordedAt: fixed.Add(-time.Hour)}, + {ID: "b", RecordedAt: fixed}, + {ID: "c", RecordedAt: fixed.Add(-2 * time.Hour)}, + } + sortItems(items) + if items[0].ID != "b" || items[1].ID != "a" || items[2].ID != "c" { + t.Fatalf("sorted = %v %v %v", items[0].ID, items[1].ID, items[2].ID) + } +} + +type failingSource struct { + fakeSource + listErr error + fetchErr error +} + +func (f *failingSource) List(context.Context) ([]Item, error) { return f.items, f.listErr } +func (f *failingSource) Fetch(context.Context, Item) (Content, error) { + return f.body, f.fetchErr +} + +func TestServiceDeliverFailures(t *testing.T) { + src := &failingSource{fakeSource: fakeSource{items: []Item{{Source: "plaud", ID: "rec-1", Title: "one"}}}, fetchErr: errors.New("boom")} + dest := &recordingDest{name: DestSpekRFC, res: Result{Note: "delegated to agent x"}} + svc := newTestService(t, config.InboxConfig{Spek: config.InboxSpek{Repo: "o/r"}}, src, dest) + + // Fetch error is recorded as failed. + items, _ := svc.Items(context.Background()) + if _, err := svc.Route(context.Background(), items[0], Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if r := items[0].Routes[DestSpekRFC]; r == nil || r.Status != StatusFailed || !strings.Contains(r.Error, "boom") { + t.Fatalf("fetch failure = %+v", r) + } + + // Unknown source is recorded as failed. + ghost := Item{Source: "ghost", ID: "g-1"} + if _, err := svc.Route(context.Background(), ghost, Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + if r, ok := svc.Store.Route(ghost.Key(), DestSpekRFC); !ok || r.Status != StatusFailed || !strings.Contains(r.Error, "not configured") { + t.Fatalf("ghost route = %+v %v", r, ok) + } + + // A note-only result (no URL) becomes delegated. + src.fetchErr = nil + if _, err := svc.Route(context.Background(), items[0], Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + if r, ok := svc.Store.Route(items[0].Key(), DestSpekRFC); !ok || r.Status != StatusDelegated || r.Error != "delegated to agent x" { + t.Fatalf("delegated route = %+v", r) + } + + // Items surfaces list errors but still returns items. + src.listErr = errors.New("list down") + items, err := svc.Items(context.Background()) + if err == nil || !strings.Contains(err.Error(), "list down") || len(items) != 1 { + t.Fatalf("items with list error = %d %v", len(items), err) + } + if _, err := svc.Sweep(context.Background()); err == nil { + t.Fatal("sweep should surface the list error") + } + if known := svc.Store.Known(); len(known) != 2 || known[0] != ghost.Key() { + t.Fatalf("known = %v", known) + } +} + +func TestStoreEdgeCases(t *testing.T) { + fixed := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + mem := NewStore("") + if mem.now().IsZero() { + t.Fatal("now") + } + mem.Now = func() time.Time { return fixed } + if st, err := mem.Begin("plaud/x", DestDibs, ""); err != nil || st.UpdatedAt != fixed { + t.Fatalf("in-memory begin = %+v %v", st, err) + } + if _, ok := mem.Route("plaud/x", DestSpekRFC); ok { + t.Fatal("route for another destination should be absent") + } + if _, ok := mem.Route("nope", DestDibs); ok { + t.Fatal("route for unknown key should be absent") + } + + // Corrupt file is tolerated; unwritable dir surfaces an error. + dir := t.TempDir() + bad := filepath.Join(dir, "bad.json") + if err := os.WriteFile(bad, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if known := NewStore(bad).Known(); len(known) != 0 { + t.Fatalf("corrupt store known = %v", known) + } + blocker := filepath.Join(dir, "file") + if err := os.WriteFile(blocker, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewStore(filepath.Join(blocker, "inbox.json")).Begin("k", DestDibs, ""); err == nil { + t.Fatal("save under a file should fail") + } +} + +func TestFactSourceEdgeCases(t *testing.T) { + src := &FactSource{ConnectorsDir: filepath.Join(t.TempDir(), "missing"), SourceType: "wisprflow"} + if src.Type() != "wisprflow" { + t.Fatal("type") + } + if items, err := src.List(context.Background()); err != nil || items != nil { + t.Fatalf("missing dir = %v %v", items, err) + } + + dir := t.TempDir() + src.ConnectorsDir = dir + if err := os.WriteFile(filepath.Join(dir, "stray.txt"), nil, 0o600); err != nil { + t.Fatal(err) + } + writeFact(t, dir, "personal", "wisprflow-w-1.md", strings.Replace(plaudFact, "source: plaud", "source: wisprflow", 1)) + writeFact(t, dir, "personal", "wisprflow-w-2.md", "no front matter at all") + writeFact(t, dir, "personal", "wisprflow-w-3.md", "---\nsource: wisprflow\nunterminated") + writeFact(t, dir, "personal", "wisprflow-noid.md", "---\nsource: wisprflow\n---\nbody") + if err := os.Mkdir(filepath.Join(dir, "personal", "wisprflow-dir.md"), 0o755); err != nil { + t.Fatal(err) + } + items, err := src.List(context.Background()) + if err != nil || len(items) != 1 || items[0].Source != "wisprflow" { + t.Fatalf("items = %+v %v", items, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := src.List(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled list = %v", err) + } + + if _, err := src.Fetch(context.Background(), Item{FactPath: filepath.Join(dir, "nope.md")}); err == nil { + t.Fatal("fetch of a missing fact should fail") + } + // Fetch falls back to the item title when the fact has none. + path := filepath.Join(dir, "personal", "wisprflow-untitled.md") + if err := os.WriteFile(path, []byte("---\nsource: wisprflow\nsource_id: u\n---\nhello"), 0o600); err != nil { + t.Fatal(err) + } + c, err := src.Fetch(context.Background(), Item{ID: "u", Title: "from item", FactPath: path}) + if err != nil || c.Title != "from item" || c.Markdown != "hello" || c.Provenance != "wisprflow://recordings/u" { + t.Fatalf("fetch = %+v %v", c, err) + } + + fm, body := splitFrontMatter("---\na: 1\n---") + if fm["a"] != "1" || body != "" { + t.Fatalf("front matter without trailing newline = %v %q", fm, body) + } +} + +func TestSpekDestinationRealExecAndLookPath(t *testing.T) { + d := &SpekDestination{Binary: "definitely-not-a-binary-" + t.Name(), DefaultRepo: "o/r"} + it := Item{Source: "plaud", ID: "r", Title: "RFC: x"} + _, err := d.Deliver(context.Background(), it, Content{Transcript: "hi"}, Request{}) + if err == nil || !strings.Contains(err.Error(), "not installed") { + t.Fatalf("missing binary = %v", err) + } + if d.lookPath("sh") != nil { + t.Skip("sh not on PATH") + } + out, err := d.exec(context.Background(), "sh", []string{"-c", "cat; echo done"}, os.Environ(), "input\n") + if err != nil || !strings.Contains(out, "input") || !strings.Contains(out, "done") { + t.Fatalf("exec = %q %v", out, err) + } + if got := lastLines("a\nb\nc\n", 2); got != "b\nc" { + t.Fatalf("lastLines = %q", got) + } + if got := (&SpekDestination{Prefixes: []string{"X:"}}).prefixes(); len(got) != 1 || got[0] != "X:" { + t.Fatalf("prefixes = %v", got) + } +} + +func TestSpekDestinationErrorPaths(t *testing.T) { + it := Item{Source: "plaud", ID: "r", Title: "RFC: x"} + ok := func(string) error { return nil } + cases := []struct { + name string + d *SpekDestination + c Content + want string + }{ + {"no repo", &SpekDestination{LookPath: ok}, Content{Transcript: "t"}, ErrNoRepo.Error()}, + {"empty", &SpekDestination{LookPath: ok, DefaultRepo: "o/r"}, Content{}, "no transcript"}, + {"token", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Token: func(context.Context, string) (string, error) { return "", errors.New("mint") }}, Content{Transcript: "t"}, "github token: mint"}, + {"exec", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Exec: func(context.Context, string, []string, []string, string) (string, error) { + return "l1\nl2\nboom", errors.New("exit 2") + }}, Content{Transcript: "t"}, "boom"}, + {"parse", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Exec: func(context.Context, string, []string, []string, string) (string, error) { + return "nothing useful", nil + }}, Content{Transcript: "t"}, "spek_rfc:"}, + {"kick fails", &SpekDestination{LookPath: func(string) error { return errors.New("nope") }, DefaultRepo: "o/r", Agent: "a", Kick: func(string, string) error { return errors.New("kick") }}, Content{Transcript: "t"}, "kick to a failed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.d.Deliver(context.Background(), it, tc.c, Request{}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + }) + } +} + +func TestNewServiceDefaultsLogger(t *testing.T) { + svc := NewService(config.InboxConfig{}, NewStore(""), nil, nil, nil) + if svc.Logger == nil { + t.Fatal("nil logger should default") + } + svc.Logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) + if items, err := svc.Items(context.Background()); err != nil || len(items) != 0 { + t.Fatalf("items with no sources = %v %v", items, err) + } +} diff --git a/src/pkg/inbox/factsource.go b/src/pkg/inbox/factsource.go new file mode 100644 index 0000000000..5785290aac --- /dev/null +++ b/src/pkg/inbox/factsource.go @@ -0,0 +1,209 @@ +package inbox + +import ( + "context" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// excerptRunes bounds the list excerpt. +const excerptRunes = 240 + +// FactSource lists items of one connector type by reading the facts that +// type's connectors wrote under <ConnectorsDir>/<layer>/. It needs nothing +// from the vendor: any connector whose facts carry `source`/`source_id` +// front matter is an Inbox source. +type FactSource struct { + // ConnectorsDir is <knowledge base>/connectors. + ConnectorsDir string + // SourceType is the connector type (plaud, wisprflow, ...). + SourceType string +} + +// Type implements Source. +func (f *FactSource) Type() string { return f.SourceType } + +// List implements Source: newest first. +func (f *FactSource) List(ctx context.Context) ([]Item, error) { + layers, err := os.ReadDir(f.ConnectorsDir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []Item + for _, layer := range layers { + if !layer.IsDir() { + continue + } + dir := filepath.Join(f.ConnectorsDir, layer.Name()) + files, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, fi := range files { + if ctx.Err() != nil { + return out, ctx.Err() + } + if fi.IsDir() || !strings.HasSuffix(fi.Name(), ".md") || !strings.HasPrefix(fi.Name(), f.SourceType+"-") { + continue + } + path := filepath.Join(dir, fi.Name()) + data, err := os.ReadFile(path) + if err != nil { + continue + } + fm, body := splitFrontMatter(string(data)) + if fm["source"] != f.SourceType || fm["source_id"] == "" || fm["status"] == "deprecated" { + continue + } + out = append(out, itemFromFact(fm, body, layer.Name(), path)) + } + } + sort.SliceStable(out, func(i, j int) bool { + if !out[i].RecordedAt.Equal(out[j].RecordedAt) { + return out[i].RecordedAt.After(out[j].RecordedAt) + } + return out[i].ID < out[j].ID + }) + return out, nil +} + +func itemFromFact(fm map[string]string, body, layer, path string) Item { + it := Item{ + Source: fm["source"], + Connector: fm["connector"], + ID: fm["source_id"], + Title: fm["title"], + FactSlug: strings.TrimSuffix(filepath.Base(path), ".md"), + FactPath: path, + Layer: layer, + Excerpt: excerpt(body), + } + if it.Title == "" { + it.Title = it.ID + } + for _, key := range []string{"attr_recorded_at", "synthesized"} { + if t, err := time.Parse(time.RFC3339, fm[key]); err == nil { + it.RecordedAt = t.UTC() + break + } + } + if ms, err := strconv.ParseInt(fm["attr_duration_ms"], 10, 64); err == nil { + it.DurationMS = ms + } + if tags := fm["attr_tags"]; tags != "" { + for _, t := range strings.Split(tags, ",") { + if t = strings.TrimSpace(t); t != "" { + it.Tags = append(it.Tags, t) + } + } + } + return it +} + +// Fetch implements Source by re-reading the fact and splitting the body +// into the connector's "## Summary" / "## Transcript" sections. +func (f *FactSource) Fetch(_ context.Context, it Item) (Content, error) { + data, err := os.ReadFile(it.FactPath) + if err != nil { + return Content{}, err + } + fm, body := splitFrontMatter(string(data)) + c := Content{ + Title: fm["title"], + Markdown: strings.TrimSpace(body), + Provenance: f.SourceType + "://recordings/" + it.ID, + } + if c.Title == "" { + c.Title = it.Title + } + c.Summary, c.Transcript = splitSections(body) + return c, nil +} + +// splitFrontMatter mirrors the connector fact layout: a `---` block of +// `key: value` lines, then the markdown body. +func splitFrontMatter(content string) (map[string]string, string) { + out := map[string]string{} + if !strings.HasPrefix(content, "---\n") { + return out, content + } + end := strings.Index(content[4:], "\n---") + if end < 0 { + return out, content + } + for _, l := range strings.Split(content[4:4+end], "\n") { + k, v, ok := strings.Cut(l, ":") + if !ok { + continue + } + out[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), "\"'") + } + rest := content[4+end+4:] + if i := strings.Index(rest, "\n"); i >= 0 { + rest = rest[i+1:] + } else { + rest = "" + } + return out, rest +} + +// splitSections returns the text under a "## Summary…" heading and under +// "## Transcript". Headings may carry a suffix such as "(Plaud)". +func splitSections(body string) (summary, transcript string) { + var cur *string + var sb, tb strings.Builder + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, "## ") { + h := strings.ToLower(line[3:]) + switch { + case strings.HasPrefix(h, "summary"): + cur = &summary + case strings.HasPrefix(h, "transcript"): + cur = &transcript + default: + cur = nil + } + continue + } + if cur == &summary { + sb.WriteString(line) + sb.WriteByte('\n') + } else if cur == &transcript { + tb.WriteString(line) + tb.WriteByte('\n') + } + } + return strings.TrimSpace(sb.String()), strings.TrimSpace(tb.String()) +} + +// excerpt returns the first prose lines of body, bounded to excerptRunes. +func excerpt(body string) string { + var b strings.Builder + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "**") { + continue + } + if b.Len() > 0 { + b.WriteByte(' ') + } + b.WriteString(line) + if utf8.RuneCountInString(b.String()) >= excerptRunes { + break + } + } + s := b.String() + if utf8.RuneCountInString(s) > excerptRunes { + r := []rune(s) + s = strings.TrimSpace(string(r[:excerptRunes])) + "…" + } + return s +} diff --git a/src/pkg/inbox/inbox.go b/src/pkg/inbox/inbox.go new file mode 100644 index 0000000000..bc6e194e16 --- /dev/null +++ b/src/pkg/inbox/inbox.go @@ -0,0 +1,123 @@ +// Package inbox turns voice-recorder items that knowledge connectors already +// ingest as facts (Plaud, Wispr Flow, ...) into a source-agnostic triage list +// with routes to spek RFCs, dibs ideas, or "knowledge only". +package inbox + +import ( + "context" + "time" +) + +// Route statuses for one (item, destination) pair. +const ( + StatusQueued = "queued" + StatusRunning = "running" + StatusFiled = "filed" + StatusFailed = "failed" +) + +// Destination names. Knowledge is implicit: items are Knowledge facts. +const ( + DestSpekRFC = "spek_rfc" + DestDibs = "dibs" + DestDismiss = "dismiss" +) + +// Item is one incoming recording/dictation as the Inbox lists it. +type Item struct { + // Source is the connector type (plaud, wisprflow, ...). + Source string `json:"source"` + // Connector is the knowledge.connectors[] entry name that synced it. + Connector string `json:"connector"` + // ID is the upstream id (Plaud file id, ...). (Source, ID) is the key. + ID string `json:"id"` + Title string `json:"title"` + Excerpt string `json:"excerpt,omitempty"` + RecordedAt time.Time `json:"recorded_at,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + Tags []string `json:"tags,omitempty"` + // FactSlug / FactPath / Layer locate the Knowledge fact backing the item. + FactSlug string `json:"fact_slug"` + FactPath string `json:"fact_path,omitempty"` + Layer string `json:"layer,omitempty"` + // Routes holds per-destination state; absent means never routed. + Routes map[string]*RouteState `json:"routes,omitempty"` + // Dismissed marks "knowledge only": the item stays a fact, no further route. + Dismissed bool `json:"dismissed,omitempty"` + // Processed is true once any route filed or the item was dismissed. + Processed bool `json:"processed"` +} + +// Key returns the stable "source/id" key used by the state store. +func (it Item) Key() string { return Key(it.Source, it.ID) } + +// Key joins a source and item id. +func Key(source, id string) string { return source + "/" + id } + +// RouteState is the lifecycle of one item on one destination. +type RouteState struct { + Status string `json:"status"` + // Repo is the owner/repo a spek_rfc route targeted. + Repo string `json:"repo,omitempty"` + // URL is the RFC issue or dibs idea once filed. + URL string `json:"url,omitempty"` + // Children are the RFC's child issue URLs. + Children []string `json:"children,omitempty"` + Error string `json:"error,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Content is the full item body a destination needs. +type Content struct { + Title string + Summary string + Transcript string + // Markdown is the whole fact body when Summary/Transcript are not split. + Markdown string + // Provenance is a stable reference such as plaud://recordings/<id>. + Provenance string +} + +// Text returns transcript+summary, falling back to the raw markdown. +func (c Content) Text() string { + if c.Transcript == "" && c.Summary == "" { + return c.Markdown + } + out := c.Transcript + if c.Summary != "" { + if out != "" { + out += "\n\n" + } + out += "Summary:\n" + c.Summary + } + return out +} + +// Source lists and fetches items of one type. The fact-backed source covers +// every connector type; vendor-specific sources can replace it later. +type Source interface { + Type() string + List(ctx context.Context) ([]Item, error) + Fetch(ctx context.Context, it Item) (Content, error) +} + +// Request is one routing request. +type Request struct { + Destination string + // Repo is the owner/repo for spek_rfc; empty uses the configured default. + Repo string +} + +// Result is what a destination filed. +type Result struct { + URL string + Children []string + // Note is set instead of URL when the work was delegated to an agent. + Note string +} + +// Destination files an item somewhere outside Knowledge. +type Destination interface { + Name() string + Deliver(ctx context.Context, it Item, c Content, req Request) (Result, error) +} diff --git a/src/pkg/inbox/inbox_test.go b/src/pkg/inbox/inbox_test.go new file mode 100644 index 0000000000..34868adcaa --- /dev/null +++ b/src/pkg/inbox/inbox_test.go @@ -0,0 +1,398 @@ +package inbox + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +const plaudFact = `--- +title: RFC: Voice inbox +type: reference +layer: personal +status: active +tags: [connector, plaud, plaud] +source: plaud +connector: plaud +source_id: rec-1 +source_path: Plaud +attr_duration_ms: 125000 +attr_plaud_file_id: rec-1 +attr_recorded_at: 2026-11-10T08:58:00Z +attr_tags: voice, ideas +synthesized: 2026-11-10T08:58:00Z +synced_at: 2026-11-11T20:00:00Z +--- + +# RFC: Voice inbox + +**Recorded:** 2026-11-10T08:58:00Z + +## Summary (Plaud) + +- ship the inbox + +## Transcript + +[00:02] Andy: Hello hive +[01:05] Andy: Second line +` + +func writeFact(t *testing.T, dir, layer, name, content string) string { + t.Helper() + p := filepath.Join(dir, layer, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func TestFactSourceListsAndFetchesConnectorFacts(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + writeFact(t, dir, "personal", "plaud-plaud-rec-0.md", strings.Replace(strings.Replace(plaudFact, "rec-1", "rec-0", -1), "status: active", "status: deprecated", 1)) + writeFact(t, dir, "personal", "notion-docs-x.md", strings.Replace(plaudFact, "source: plaud", "source: notion", 1)) + src := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + items, err := src.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 { + t.Fatalf("items = %+v", items) + } + it := items[0] + if it.ID != "rec-1" || it.Source != "plaud" || it.Connector != "plaud" || it.Title != "RFC: Voice inbox" || it.FactSlug != "plaud-plaud-rec-1" || it.Layer != "personal" { + t.Fatalf("item = %+v", it) + } + if it.DurationMS != 125000 || !it.RecordedAt.Equal(time.Date(2026, 11, 10, 8, 58, 0, 0, time.UTC)) || strings.Join(it.Tags, ",") != "voice,ideas" { + t.Fatalf("attrs = %+v", it) + } + if !strings.Contains(it.Excerpt, "ship the inbox") || strings.Contains(it.Excerpt, "Recorded:") { + t.Fatalf("excerpt = %q", it.Excerpt) + } + c, err := src.Fetch(context.Background(), it) + if err != nil { + t.Fatal(err) + } + if c.Summary != "- ship the inbox" || !strings.HasPrefix(c.Transcript, "[00:02] Andy: Hello hive") || c.Provenance != "plaud://recordings/rec-1" { + t.Fatalf("content = %+v", c) + } +} + +func TestRuleMatching(t *testing.T) { + it := Item{Source: "plaud", Title: "rfc: Build the thing", Tags: []string{"Ideas"}} + cases := []struct { + name string + rule Rule + want bool + }{ + {"prefix case-insensitive", Rule{TitlePrefix: "RFC:"}, true}, + {"prefix mismatch", Rule{TitlePrefix: "Hive:"}, false}, + {"tag", Rule{Tag: "ideas"}, true}, + {"source mismatch", Rule{Source: "wisprflow", TitlePrefix: "RFC:"}, false}, + {"source match", Rule{Source: "plaud", TitlePrefix: "RFC:"}, true}, + {"no filter never matches", Rule{Destination: DestDibs}, false}, + } + for _, tc := range cases { + if got := RuleMatches(tc.rule, it); got != tc.want { + t.Errorf("%s: got %v want %v", tc.name, got, tc.want) + } + } + if got := StripPrefix("RFC: Build the thing", DefaultTitlePrefixes); got != "Build the thing" { + t.Fatalf("StripPrefix = %q", got) + } + if got := StripPrefix("Plain", DefaultTitlePrefixes); got != "Plain" { + t.Fatalf("StripPrefix = %q", got) + } +} + +func TestStoreTransitions(t *testing.T) { + st := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + key := Key("plaud", "rec-1") + if _, err := st.Begin(key, DestSpekRFC, "o/r"); err != nil { + t.Fatal(err) + } + if _, err := st.Begin(key, DestSpekRFC, "o/r"); !errors.Is(err, ErrRouteBusy) { + t.Fatalf("second Begin while queued: %v", err) + } + if _, err := st.Set(key, DestSpekRFC, StatusRunning, Result{}, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Begin(key, DestSpekRFC, "o/r"); !errors.Is(err, ErrRouteBusy) { + t.Fatalf("Begin while running: %v", err) + } + // dibs route on the same item is independent. + if _, err := st.Begin(key, DestDibs, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Set(key, DestSpekRFC, StatusFiled, Result{URL: "https://github.com/o/r/issues/1", Children: []string{"https://github.com/o/r/issues/2"}}, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Set(key, DestDibs, StatusFailed, Result{}, "boom"); err != nil { + t.Fatal(err) + } + // Reload from disk. + st2 := NewStore(st.Path) + items := []Item{{Source: "plaud", ID: "rec-1"}, {Source: "plaud", ID: "rec-9"}} + st2.Apply(items) + r := items[0].Routes[DestSpekRFC] + if r == nil || r.Status != StatusFiled || r.URL != "https://github.com/o/r/issues/1" || len(r.Children) != 1 || r.Repo != "o/r" { + t.Fatalf("spek route = %+v", r) + } + if d := items[0].Routes[DestDibs]; d == nil || d.Status != StatusFailed || d.Error != "boom" { + t.Fatalf("dibs route = %+v", d) + } + if !items[0].Processed || items[1].Processed { + t.Fatalf("processed flags = %v %v", items[0].Processed, items[1].Processed) + } + // A failed route may be retried; a filed one may be re-filed. + if _, err := st2.Begin(key, DestDibs, ""); err != nil { + t.Fatalf("retry after failure: %v", err) + } + if err := st2.Dismiss(Key("plaud", "rec-9"), true); err != nil { + t.Fatal(err) + } + st2.Apply(items) + if !items[1].Processed || !items[1].Dismissed { + t.Fatalf("dismissed item must be processed: %+v", items[1]) + } +} + +func TestParsePublishOutput(t *testing.T) { + jsonOut := `Publishing... +{"target":"github","repo":"o/r","rfc_url":"https://github.com/o/r/issues/10","rfc_number":10,"children":[{"id":"a","url":"https://github.com/o/r/issues/11","number":11},{"id":"b","url":"https://github.com/o/r/issues/12","number":12}],"created":3}` + res, err := ParsePublishOutput(jsonOut) + if err != nil { + t.Fatal(err) + } + if res.URL != "https://github.com/o/r/issues/10" || len(res.Children) != 2 || res.Children[1] != "https://github.com/o/r/issues/12" { + t.Fatalf("json result = %+v", res) + } + textOut := "Draft written to .spektacular/specs/voice.md\nRFC: https://github.com/o/r/issues/20\n child: https://github.com/o/r/issues/21\n child: https://github.com/o/r/issues/22 (dup) https://github.com/o/r/issues/22\n" + res, err = ParsePublishOutput(textOut) + if err != nil { + t.Fatal(err) + } + if res.URL != "https://github.com/o/r/issues/20" || len(res.Children) != 2 { + t.Fatalf("text result = %+v", res) + } + if _, err := ParsePublishOutput("error: nothing happened"); err == nil { + t.Fatal("no URL must be an error") + } +} + +func TestRenderAgentPrompt(t *testing.T) { + it := Item{Source: "plaud", ID: "rec-1"} + c := Content{Summary: "- ship it", Transcript: "Hello hive", Provenance: "plaud://recordings/rec-1"} + p := RenderAgentPrompt(it, c, "o/r", "Voice inbox v2") + for _, want := range []string{"spektacular publish voice-inbox-v2 --repo o/r --allow-draft --format json", "spektacular init", "Open questions", "plaud://recordings/rec-1", "## Summary\n- ship it", "## Transcript\nHello hive"} { + if !strings.Contains(p, want) { + t.Errorf("prompt missing %q:\n%s", want, p) + } + } +} + +type fakeSource struct { + items []Item + body Content +} + +func (f *fakeSource) Type() string { return "plaud" } +func (f *fakeSource) List(context.Context) ([]Item, error) { return f.items, nil } +func (f *fakeSource) Fetch(context.Context, Item) (Content, error) { return f.body, nil } + +func newTestService(t *testing.T, cfg config.InboxConfig, src Source, dests ...Destination) *Service { + t.Helper() + store := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + svc := NewService(cfg, store, []Source{src}, dests, slog.New(slog.NewTextHandler(os.Stderr, nil))) + svc.Run = func(fn func()) { fn() } + return svc +} + +func TestSpekDestinationRunsVoicePublish(t *testing.T) { + var gotArgs []string + var gotStdin string + var gotEnv []string + calls := 0 + d := &SpekDestination{ + Binary: "spektacular", + DefaultRepo: "o/r", + Token: func(context.Context, string) (string, error) { return "tok", nil }, + LookPath: func(string) error { return nil }, + Exec: func(_ context.Context, _ string, args, env []string, stdin string) (string, error) { + calls++ + if calls == 1 { + return "Error: unknown flag: --format", errors.New("exit status 1") + } + gotArgs, gotEnv, gotStdin = args, env, stdin + return "RFC: https://github.com/o/r/issues/5\nhttps://github.com/o/r/issues/6\n", nil + }, + } + it := Item{Source: "plaud", ID: "rec-1", Title: "RFC: Voice inbox"} + c := Content{Title: "RFC: Voice inbox", Transcript: "Hello hive", Summary: "- ship it", Provenance: "plaud://recordings/rec-1"} + res, err := d.Deliver(context.Background(), it, c, Request{Destination: DestSpekRFC}) + if err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("expected retry without --format json, calls=%d", calls) + } + want := []string{"voice", "--from", "stdin", "--title", "Voice inbox", "--publish", "--repo", "o/r"} + if strings.Join(gotArgs, " ") != strings.Join(want, " ") { + t.Fatalf("args = %v", gotArgs) + } + if !strings.Contains(gotStdin, "Hello hive") || !strings.Contains(gotStdin, "plaud://recordings/rec-1") { + t.Fatalf("stdin = %q", gotStdin) + } + hasTok := false + for _, e := range gotEnv { + if e == "GITHUB_TOKEN=tok" { + hasTok = true + } + } + if !hasTok { + t.Fatal("GITHUB_TOKEN not injected") + } + if res.URL != "https://github.com/o/r/issues/5" || len(res.Children) != 1 { + t.Fatalf("result = %+v", res) + } +} + +func TestSpekDestinationFallsBackToAgentKick(t *testing.T) { + var kicked, prompt string + d := &SpekDestination{ + LookPath: func(string) error { return errors.New("not found") }, + Agent: "builder", + Kick: func(agent, p string) error { kicked, prompt = agent, p; return nil }, + } + res, err := d.Deliver(context.Background(), Item{Source: "plaud", ID: "x", Title: "Idea"}, Content{Markdown: "stuff"}, Request{Repo: "o/r"}) + if err != nil { + t.Fatal(err) + } + if kicked != "builder" || !strings.Contains(prompt, "--repo o/r") || res.Note == "" || res.URL != "" { + t.Fatalf("kick=%q note=%q prompt=%q", kicked, res.Note, prompt) + } + d.Agent = "" + if _, err := d.Deliver(context.Background(), Item{Source: "plaud", ID: "x"}, Content{Markdown: "stuff"}, Request{Repo: "o/r"}); err == nil { + t.Fatal("missing binary without agent must fail") + } + if _, err := (&SpekDestination{LookPath: func(string) error { return nil }}).Deliver(context.Background(), Item{}, Content{Markdown: "x"}, Request{}); !errors.Is(err, ErrNoRepo) { + t.Fatalf("no repo: %v", err) + } +} + +type recordingDest struct { + name string + mu sync.Mutex + got []Request + res Result + err error +} + +func (r *recordingDest) Name() string { return r.name } +func (r *recordingDest) Deliver(_ context.Context, _ Item, _ Content, req Request) (Result, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.got = append(r.got, req) + return r.res, r.err +} + +func TestServiceRouteAndSweep(t *testing.T) { + src := &fakeSource{items: []Item{ + {Source: "plaud", ID: "rec-1", Title: "RFC: one", RecordedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}, + {Source: "plaud", ID: "rec-2", Title: "note", RecordedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, body: Content{Markdown: "x"}} + spek := &recordingDest{name: DestSpekRFC, res: Result{URL: "https://github.com/o/r/issues/1"}} + dibs := &recordingDest{name: DestDibs, err: errors.New("dibs down")} + cfg := config.InboxConfig{ + Rules: []config.InboxRule{{TitlePrefix: "RFC:", Destination: DestSpekRFC}}, + Spek: config.InboxSpek{Repo: "o/r"}, + } + svc := newTestService(t, cfg, src, spek, dibs) + + routed, err := svc.Sweep(context.Background()) + if err != nil || routed != 1 { + t.Fatalf("sweep routed=%d err=%v", routed, err) + } + if len(spek.got) != 1 || spek.got[0].Repo != "o/r" { + t.Fatalf("spek requests = %+v", spek.got) + } + items, _ := svc.Items(context.Background()) + if items[0].ID != "rec-1" || items[0].Routes[DestSpekRFC].Status != StatusFiled || !items[0].Processed { + t.Fatalf("rec-1 = %+v", items[0]) + } + if items[1].Processed || Unprocessed(items) != 1 { + t.Fatalf("rec-2 should be unprocessed: %+v", items[1]) + } + // A second sweep must not re-fire on seen items. + if routed, _ := svc.Sweep(context.Background()); routed != 0 { + t.Fatalf("second sweep routed %d", routed) + } + // Manual dibs route fails and records the error. + if _, err := svc.Route(context.Background(), items[1], Request{Destination: DestDibs}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if r := items[1].Routes[DestDibs]; r == nil || r.Status != StatusFailed || r.Error != "dibs down" { + t.Fatalf("dibs route = %+v", r) + } + if _, err := svc.Route(context.Background(), items[1], Request{Destination: "nope"}); !errors.Is(err, ErrUnknownDestination) { + t.Fatalf("unknown destination: %v", err) + } + // Dismiss = knowledge only. + if _, err := svc.Route(context.Background(), items[1], Request{Destination: DestDismiss}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if !items[1].Dismissed || Unprocessed(items) != 0 { + t.Fatalf("dismiss failed: %+v", items[1]) + } +} + +func TestDibsDestinationPostsIdea(t *testing.T) { + var gotAuth string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/ideas" || r.Method != http.MethodPost { + t.Errorf("unexpected %s %s", r.Method, r.URL.Path) + } + gotAuth = r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"idea-7","title":"x"}`)) + })) + defer srv.Close() + d := &DibsDestination{URL: srv.URL + "/", Token: "hub-tok"} + it := Item{Source: "plaud", ID: "rec-1", Title: "Idea", RecordedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)} + res, err := d.Deliver(context.Background(), it, Content{Title: "Idea", Summary: "sum", Transcript: "words", Provenance: "plaud://recordings/rec-1"}, Request{}) + if err != nil { + t.Fatal(err) + } + if gotAuth != "Bearer hub-tok" || gotBody["title"] != "Idea" || gotBody["visibility"] != "private" || gotBody["status"] != "draft" { + t.Fatalf("auth=%q body=%v", gotAuth, gotBody) + } + body, _ := gotBody["body"].(string) + if !strings.Contains(body, "plaud://recordings/rec-1") || !strings.Contains(body, "## Transcript") || !strings.HasPrefix(body, "sum") { + t.Fatalf("body = %q", body) + } + if res.URL != srv.URL+"/api/ideas/idea-7" { + t.Fatalf("url = %q", res.URL) + } + if _, err := (&DibsDestination{}).Deliver(context.Background(), it, Content{}, Request{}); !errors.Is(err, ErrDibsUnconfigured) { + t.Fatalf("unconfigured: %v", err) + } +} diff --git a/src/pkg/inbox/rules.go b/src/pkg/inbox/rules.go new file mode 100644 index 0000000000..61adf1a198 --- /dev/null +++ b/src/pkg/inbox/rules.go @@ -0,0 +1,62 @@ +package inbox + +import ( + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// DefaultTitlePrefixes are the prefixes the UI offers when a rule has none. +var DefaultTitlePrefixes = []string{"RFC:", "Hive:"} + +// Rule is one auto-routing rule (mirrors config.InboxRule). +type Rule = config.InboxRule + +// RuleMatches reports whether r applies to it: source (when set) must equal +// the item source, and at least one of title_prefix / tag must match. +// A rule with neither filter never matches, so a bad config cannot route +// every recording. +func RuleMatches(r Rule, it Item) bool { + if src := strings.TrimSpace(r.Source); src != "" && !strings.EqualFold(src, it.Source) { + return false + } + prefix := strings.TrimSpace(r.TitlePrefix) + tag := strings.TrimSpace(r.Tag) + if prefix == "" && tag == "" { + return false + } + if prefix != "" && strings.HasPrefix(strings.ToLower(strings.TrimSpace(it.Title)), strings.ToLower(prefix)) { + return true + } + if tag != "" { + for _, t := range it.Tags { + if strings.EqualFold(strings.TrimSpace(t), tag) { + return true + } + } + } + return false +} + +// FirstRule returns the first rule matching it. +func FirstRule(rules []Rule, it Item) (Rule, bool) { + for _, r := range rules { + if RuleMatches(r, it) { + return r, true + } + } + return Rule{}, false +} + +// StripPrefix removes a matched routing prefix ("RFC: Foo" → "Foo") so the +// filed title reads naturally; unmatched titles are returned trimmed. +func StripPrefix(title string, prefixes []string) string { + t := strings.TrimSpace(title) + for _, p := range prefixes { + p = strings.TrimSpace(p) + if p != "" && len(t) > len(p) && strings.EqualFold(t[:len(p)], p) { + return strings.TrimSpace(t[len(p):]) + } + } + return t +} diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go new file mode 100644 index 0000000000..ff1bcc3c92 --- /dev/null +++ b/src/pkg/inbox/service.go @@ -0,0 +1,279 @@ +package inbox + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// routeTimeout bounds one background delivery. +const routeTimeout = 15 * time.Minute + +// Settings is the operator-editable part of the Inbox (mirrors inbox.* +// YAML minus binary/agent plumbing). +type Settings struct { + Sources []string `json:"sources"` + Rules []config.InboxRule `json:"rules"` + SpekRepo string `json:"spek_repo"` + DibsURL string `json:"dibs_url"` + // DibsTokenSet reports whether inbox.dibs.token_env resolves; the token + // itself is never returned. + DibsTokenSet bool `json:"dibs_token_set"` + SpekReady bool `json:"spek_ready"` +} + +// Service lists items across sources, applies persisted state, runs +// auto-routing rules and executes manual routes in the background. +type Service struct { + Store *Store + Logger *slog.Logger + // Now is injectable for tests. + Now func() time.Time + + mu sync.RWMutex + cfg config.InboxConfig + sources map[string]Source + dests map[string]Destination + wg sync.WaitGroup + // Run executes a delivery; nil runs it in a goroutine. Tests set it to + // run inline. + Run func(fn func()) + // Status snapshots for the settings view. + spekReady func() bool + dibsTokenSet func() bool +} + +// NewService builds a service for cfg with the given sources/destinations. +func NewService(cfg config.InboxConfig, store *Store, sources []Source, dests []Destination, logger *slog.Logger) *Service { + if logger == nil { + logger = slog.Default() + } + s := &Service{Store: store, Logger: logger, cfg: cfg, sources: map[string]Source{}, dests: map[string]Destination{}} + for _, src := range sources { + s.sources[src.Type()] = src + } + for _, d := range dests { + s.dests[d.Name()] = d + } + return s +} + +// SetReadiness installs the probes behind Settings.SpekReady / DibsTokenSet. +func (s *Service) SetReadiness(spek, dibs func() bool) { + s.spekReady, s.dibsTokenSet = spek, dibs +} + +// Config returns the current inbox config. +func (s *Service) Config() config.InboxConfig { + s.mu.RLock() + defer s.mu.RUnlock() + return s.cfg +} + +// UpdateConfig replaces the routing config (after the dashboard saved it). +func (s *Service) UpdateConfig(cfg config.InboxConfig) { + s.mu.Lock() + s.cfg = cfg + s.mu.Unlock() +} + +// Settings returns the operator view of the config. +func (s *Service) Settings() Settings { + cfg := s.Config() + st := Settings{ + Sources: cfg.EffectiveSources(), + Rules: cfg.Rules, + SpekRepo: cfg.Spek.Repo, + DibsURL: cfg.Dibs.URL, + } + if st.Rules == nil { + st.Rules = []config.InboxRule{} + } + if s.spekReady != nil { + st.SpekReady = s.spekReady() + } + if s.dibsTokenSet != nil { + st.DibsTokenSet = s.dibsTokenSet() + } + return st +} + +// Items lists every item across the configured sources with state applied, +// newest first. +func (s *Service) Items(ctx context.Context) ([]Item, error) { + cfg := s.Config() + var out []Item + var errs []error + for _, typ := range cfg.EffectiveSources() { + src := s.source(typ) + if src == nil { + continue + } + items, err := src.List(ctx) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", typ, err)) + } + out = append(out, items...) + } + s.Store.Apply(out) + sortItems(out) + return out, errors.Join(errs...) +} + +func (s *Service) source(typ string) Source { + s.mu.RLock() + defer s.mu.RUnlock() + return s.sources[typ] +} + +// Find returns one item by key. +func (s *Service) Find(ctx context.Context, source, id string) (Item, bool, error) { + items, err := s.Items(ctx) + for _, it := range items { + if it.Source == source && it.ID == id { + return it, true, err + } + } + return Item{}, false, err +} + +// Unprocessed counts items with no filed route and not dismissed. +func Unprocessed(items []Item) int { + n := 0 + for _, it := range items { + if !it.Processed { + n++ + } + } + return n +} + +// Dismiss marks an item knowledge-only. +func (s *Service) Dismiss(source, id string, dismissed bool) error { + return s.Store.Dismiss(Key(source, id), dismissed) +} + +// ErrUnknownDestination is returned for a destination the service lacks. +var ErrUnknownDestination = errors.New("unknown destination") + +// Route queues it for dest and runs the delivery in the background. The +// returned state is the queued state; callers poll Items for progress. +func (s *Service) Route(ctx context.Context, it Item, req Request) (RouteState, error) { + if req.Destination == DestDismiss { + if err := s.Dismiss(it.Source, it.ID, true); err != nil { + return RouteState{}, err + } + return RouteState{Status: StatusFiled, UpdatedAt: s.now()}, nil + } + s.mu.RLock() + dest := s.dests[req.Destination] + s.mu.RUnlock() + if dest == nil { + return RouteState{}, fmt.Errorf("%w: %q", ErrUnknownDestination, req.Destination) + } + if req.Destination == DestSpekRFC && strings.TrimSpace(req.Repo) == "" { + req.Repo = s.Config().Spek.Repo + } + st, err := s.Store.Begin(it.Key(), req.Destination, req.Repo) + if err != nil { + return st, err + } + s.wg.Add(1) + run := s.Run + if run == nil { + run = func(fn func()) { go fn() } + } + run(func() { + defer s.wg.Done() + s.deliver(it, dest, req) + }) + return st, nil +} + +func (s *Service) deliver(it Item, dest Destination, req Request) { + key := it.Key() + ctx, cancel := context.WithTimeout(context.Background(), routeTimeout) + defer cancel() + if _, err := s.Store.Set(key, req.Destination, StatusRunning, Result{}, ""); err != nil { + s.Logger.Warn("inbox: persist running state", "item", key, "error", err) + } + src := s.source(it.Source) + if src == nil { + s.fail(key, req.Destination, fmt.Errorf("source %q is not configured", it.Source)) + return + } + content, err := src.Fetch(ctx, it) + if err != nil { + s.fail(key, req.Destination, fmt.Errorf("fetch: %w", err)) + return + } + res, err := dest.Deliver(ctx, it, content, req) + if err != nil { + s.fail(key, req.Destination, err) + return + } + status := StatusFiled + if res.URL == "" && res.Note != "" { + status = StatusDelegated + } + if _, err := s.Store.Set(key, req.Destination, status, res, res.Note); err != nil { + s.Logger.Warn("inbox: persist filed state", "item", key, "error", err) + } + s.Logger.Info("inbox: routed", "item", key, "destination", req.Destination, "status", status, "url", res.URL) +} + +func (s *Service) fail(key, dest string, err error) { + s.Logger.Warn("inbox: route failed", "item", key, "destination", dest, "error", err) + if _, perr := s.Store.Set(key, dest, StatusFailed, Result{}, err.Error()); perr != nil { + s.Logger.Warn("inbox: persist failed state", "item", key, "error", perr) + } +} + +// Sweep applies auto-routing rules to items the store has never seen and +// marks every listed item seen, so a rule added later never back-fires on +// old recordings. It is called after each connector sync. +func (s *Service) Sweep(ctx context.Context) (routed int, err error) { + items, lerr := s.Items(ctx) + rules := s.Config().Rules + for _, it := range items { + if s.Store.Seen(it.Key()) { + continue + } + rule, ok := FirstRule(rules, it) + if !ok { + _ = s.Store.MarkSeen(it.Key()) + continue + } + if _, rerr := s.Route(ctx, it, Request{Destination: rule.Destination, Repo: rule.Repo}); rerr != nil { + s.Logger.Warn("inbox: auto-route", "item", it.Key(), "error", rerr) + _ = s.Store.MarkSeen(it.Key()) + continue + } + routed++ + } + return routed, lerr +} + +// Wait blocks until background deliveries finish (tests, shutdown). +func (s *Service) Wait() { s.wg.Wait() } + +func (s *Service) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +func sortItems(items []Item) { + for i := 1; i < len(items); i++ { + for j := i; j > 0 && items[j].RecordedAt.After(items[j-1].RecordedAt); j-- { + items[j], items[j-1] = items[j-1], items[j] + } + } +} diff --git a/src/pkg/inbox/spek.go b/src/pkg/inbox/spek.go new file mode 100644 index 0000000000..4a0e39df73 --- /dev/null +++ b/src/pkg/inbox/spek.go @@ -0,0 +1,257 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "text/template" + "time" +) + +// StatusDelegated means the route was handed to a hive agent (no binary in +// the image); the agent files the RFC and the operator links it manually. +const StatusDelegated = "delegated" + +// spekTimeout bounds one `spektacular voice --publish` run. +const spekTimeout = 10 * time.Minute + +// Kicker sends a prompt to a named hive agent (the dashboard's kick path). +type Kicker func(agent, prompt string) error + +// SpekDestination files an item as a spek RFC + child issues by running +// `spektacular voice --from stdin --publish` inside the hive. When the +// binary is missing and Agent is set, it kicks that agent with a prompt +// instead (fallback until the hive image ships spektacular). +type SpekDestination struct { + Binary string + DefaultRepo string + // Token mints a GitHub token for repo; empty/nil inherits the process env. + Token func(ctx context.Context, repo string) (string, error) + // Exec runs the command; nil uses os/exec. Tests inject it. + Exec func(ctx context.Context, binary string, args []string, env []string, stdin string) (string, error) + // LookPath reports whether binary is runnable; nil uses exec.LookPath. + LookPath func(binary string) error + Agent string + Kick Kicker + Prefixes []string +} + +// Name implements Destination. +func (d *SpekDestination) Name() string { return DestSpekRFC } + +// ErrNoRepo is returned when neither the request nor config names a repo. +var ErrNoRepo = errors.New("spek_rfc: no repo given and inbox.spek.repo is unset") + +// Deliver implements Destination. +func (d *SpekDestination) Deliver(ctx context.Context, it Item, c Content, req Request) (Result, error) { + repo := strings.TrimSpace(req.Repo) + if repo == "" { + repo = strings.TrimSpace(d.DefaultRepo) + } + if repo == "" { + return Result{}, ErrNoRepo + } + title := StripPrefix(firstNonEmpty(c.Title, it.Title), d.prefixes()) + text := c.Text() + if strings.TrimSpace(text) == "" { + return Result{}, fmt.Errorf("spek_rfc: item %s has no transcript or summary", it.Key()) + } + binary := firstNonEmpty(d.Binary, "spektacular") + if err := d.lookPath(binary); err != nil { + if d.Agent != "" && d.Kick != nil { + prompt := RenderAgentPrompt(it, c, repo, title) + if kerr := d.Kick(d.Agent, prompt); kerr != nil { + return Result{}, fmt.Errorf("spek_rfc: %s unavailable and kick to %s failed: %w", binary, d.Agent, kerr) + } + return Result{Note: "delegated to agent " + d.Agent}, nil + } + return Result{}, fmt.Errorf("spek_rfc: %s is not installed in this hive (set inbox.spek.binary or inbox.spek.agent): %w", binary, err) + } + stdin := text + "\n\nProvenance: " + c.Provenance + "\n" + env := os.Environ() + if d.Token != nil { + tok, err := d.Token(ctx, repo) + if err != nil { + return Result{}, fmt.Errorf("spek_rfc: github token: %w", err) + } + if tok != "" { + env = append(env, "GITHUB_TOKEN="+tok, "GH_TOKEN="+tok) + } + } + rctx, cancel := context.WithTimeout(ctx, spekTimeout) + defer cancel() + args := []string{"voice", "--from", "stdin", "--title", title, "--publish", "--repo", repo, "--format", "json"} + out, err := d.exec(rctx, binary, args, env, stdin) + if err != nil && isUnknownFlag(out, err) { + out, err = d.exec(rctx, binary, args[:len(args)-2], env, stdin) + } + if err != nil { + return Result{}, fmt.Errorf("spek_rfc: %w: %s", err, lastLines(out, 5)) + } + res, perr := ParsePublishOutput(out) + if perr != nil { + return Result{}, fmt.Errorf("spek_rfc: %w", perr) + } + return res, nil +} + +func (d *SpekDestination) prefixes() []string { + if len(d.Prefixes) > 0 { + return d.Prefixes + } + return DefaultTitlePrefixes +} + +func (d *SpekDestination) lookPath(binary string) error { + if d.LookPath != nil { + return d.LookPath(binary) + } + _, err := exec.LookPath(binary) + return err +} + +func (d *SpekDestination) exec(ctx context.Context, binary string, args, env []string, stdin string) (string, error) { + if d.Exec != nil { + return d.Exec(ctx, binary, args, env, stdin) + } + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Env = env + cmd.Stdin = strings.NewReader(stdin) + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + return buf.String(), err +} + +func isUnknownFlag(out string, err error) bool { + s := strings.ToLower(out + " " + err.Error()) + return strings.Contains(s, "unknown flag") || strings.Contains(s, "flag provided but not defined") || strings.Contains(s, "unknown shorthand flag") +} + +// publishJSON is the `spektacular publish --format json` envelope (see +// spektacular internal/publish.Result). +type publishJSON struct { + RFCURL string `json:"rfc_url"` + Children []struct { + URL string `json:"url"` + } `json:"children"` +} + +var issueURLRe = regexp.MustCompile(`https://[^\s"'<>)]+/[^\s"'<>)/]+/[^\s"'<>)/]+/(?:issues|pull)/\d+`) + +// ParsePublishOutput accepts either the JSON envelope (`rfc_url`, +// `children[].url`) or plain text where the first issue URL is the RFC and +// the rest are children (what `spektacular voice` prints). +func ParsePublishOutput(out string) (Result, error) { + trimmed := strings.TrimSpace(out) + if start := strings.Index(trimmed, "{"); start >= 0 { + var pj publishJSON + if dec := json.NewDecoder(strings.NewReader(trimmed[start:])); dec.Decode(&pj) == nil && pj.RFCURL != "" { + res := Result{URL: pj.RFCURL} + for _, ch := range pj.Children { + if ch.URL != "" { + res.Children = append(res.Children, ch.URL) + } + } + return res, nil + } + } + urls := issueURLRe.FindAllString(out, -1) + seen := map[string]bool{} + var uniq []string + for _, u := range urls { + if !seen[u] { + seen[u] = true + uniq = append(uniq, u) + } + } + if len(uniq) == 0 { + return Result{}, fmt.Errorf("no RFC URL in spektacular output: %s", lastLines(out, 3)) + } + return Result{URL: uniq[0], Children: uniq[1:]}, nil +} + +var agentPromptTmpl = template.Must(template.New("spek").Parse(`Turn the voice recording below into a spek RFC with child issues on {{.Repo}}. + +Steps: +1. Ensure the spektacular CLI is available ("go install github.com/hivecommons/spektacular@latest" or a release binary). +2. In the {{.Repo}} clone, run "spektacular init" if .spektacular/ is missing. +3. Create a spec named "{{.Name}}" seeded from the transcript and summary below. Fill it non-interactively using your judgment; list anything unclear under "Open questions". +4. Run: spektacular publish {{.Name}} --repo {{.Repo}} --allow-draft --format json +5. Reply with the rfc_url and the child issue URLs from the JSON output. + +Provenance: {{.Provenance}} +Title: {{.Title}} + +{{if .Summary}}## Summary +{{.Summary}} + +{{end}}## Transcript +{{.Transcript}} +`)) + +// RenderAgentPrompt is the fallback kick prompt used when the hive image +// lacks the spektacular binary. +func RenderAgentPrompt(it Item, c Content, repo, title string) string { + transcript := c.Transcript + if transcript == "" { + transcript = c.Markdown + } + var buf bytes.Buffer + _ = agentPromptTmpl.Execute(&buf, map[string]string{ + "Repo": repo, + "Name": specName(title), + "Title": title, + "Summary": c.Summary, + "Transcript": transcript, + "Provenance": c.Provenance, + }) + return buf.String() +} + +// specName derives a kebab-case spec name from a title. +func specName(title string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(title) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + dash = false + } else if !dash && b.Len() > 0 { + b.WriteByte('-') + dash = true + } + } + name := strings.TrimRight(b.String(), "-") + if name == "" { + name = "voice-rfc" + } + if len(name) > 60 { + name = strings.TrimRight(name[:60], "-") + } + return name +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func lastLines(s string, n int) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} diff --git a/src/pkg/inbox/store.go b/src/pkg/inbox/store.go new file mode 100644 index 0000000000..17f3e0caec --- /dev/null +++ b/src/pkg/inbox/store.go @@ -0,0 +1,218 @@ +package inbox + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" +) + +// entryState is the persisted per-item state, keyed by Key(source, id). +type entryState struct { + Routes map[string]*RouteState `json:"routes,omitempty"` + Dismissed bool `json:"dismissed,omitempty"` +} + +// Store persists route states as one JSON file next to connector state. +type Store struct { + Path string + Now func() time.Time + + mu sync.Mutex + entries map[string]*entryState + loaded bool +} + +// NewStore returns a store backed by path (created on first save). +func NewStore(path string) *Store { + return &Store{Path: path} +} + +func (s *Store) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +func (s *Store) load() { + if s.loaded { + return + } + s.loaded = true + s.entries = map[string]*entryState{} + if s.Path == "" { + return + } + data, err := os.ReadFile(s.Path) + if err != nil { + return + } + _ = json.Unmarshal(data, &s.entries) + if s.entries == nil { + s.entries = map[string]*entryState{} + } +} + +func (s *Store) save() error { + if s.Path == "" { + return nil + } + data, err := json.MarshalIndent(s.entries, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(s.Path), 0o755); err != nil { + return err + } + tmp := s.Path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, s.Path) +} + +func (s *Store) entry(key string) *entryState { + e := s.entries[key] + if e == nil { + e = &entryState{Routes: map[string]*RouteState{}} + s.entries[key] = e + } + if e.Routes == nil { + e.Routes = map[string]*RouteState{} + } + return e +} + +// Apply copies persisted state onto items and derives Processed. +func (s *Store) Apply(items []Item) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + for i := range items { + e := s.entries[items[i].Key()] + if e == nil { + items[i].Processed = false + continue + } + items[i].Dismissed = e.Dismissed + items[i].Routes = cloneRoutes(e.Routes) + items[i].Processed = e.Dismissed + for _, r := range e.Routes { + if r.Status == StatusFiled { + items[i].Processed = true + } + } + } +} + +func cloneRoutes(in map[string]*RouteState) map[string]*RouteState { + if len(in) == 0 { + return nil + } + out := make(map[string]*RouteState, len(in)) + for k, v := range in { + c := *v + c.Children = append([]string(nil), v.Children...) + out[k] = &c + } + return out +} + +// Route returns the current state for one destination, if any. +func (s *Store) Route(key, dest string) (RouteState, bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entries[key] + if e == nil || e.Routes[dest] == nil { + return RouteState{}, false + } + return *e.Routes[dest], true +} + +// ErrRouteBusy is returned when a route is queued or running already. +var ErrRouteBusy = fmt.Errorf("route already in progress") + +// Begin transitions key/dest to queued. A queued or running route is +// rejected; filed and failed routes may be retried (filed = re-file). +func (s *Store) Begin(key, dest, repo string) (RouteState, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entry(key) + if cur := e.Routes[dest]; cur != nil && (cur.Status == StatusQueued || cur.Status == StatusRunning) { + return *cur, ErrRouteBusy + } + st := &RouteState{Status: StatusQueued, Repo: repo, UpdatedAt: s.now()} + e.Routes[dest] = st + return *st, s.save() +} + +// Set replaces the status (and optional outcome) of key/dest. +func (s *Store) Set(key, dest, status string, res Result, errText string) (RouteState, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entry(key) + st := e.Routes[dest] + if st == nil { + st = &RouteState{} + e.Routes[dest] = st + } + st.Status = status + st.UpdatedAt = s.now() + st.Error = errText + if status == StatusFiled { + st.URL = res.URL + st.Children = append([]string(nil), res.Children...) + } + return *st, s.save() +} + +// Dismiss marks the item knowledge-only (or clears that mark). +func (s *Store) Dismiss(key string, dismissed bool) error { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + s.entry(key).Dismissed = dismissed + return s.save() +} + +// Known returns every key the store has seen, sorted (for tests/debugging). +func (s *Store) Known() []string { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + out := make([]string, 0, len(s.entries)) + for k := range s.entries { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// Seen reports whether the store has any record of key; Sweep uses it so +// auto-rules only fire on items that arrived after the rule existed. +func (s *Store) Seen(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + _, ok := s.entries[key] + return ok +} + +// MarkSeen records key without any route so a later Sweep skips it. +func (s *Store) MarkSeen(key string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + if _, ok := s.entries[key]; ok { + return nil + } + s.entry(key) + return s.save() +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go index de0c27e432..f0ea586090 100644 --- a/src/pkg/knowledge/connector/plaud.go +++ b/src/pkg/knowledge/connector/plaud.go @@ -48,6 +48,12 @@ type plaudConnector struct { dial mcpDialer now func() time.Time stateDir string + // home is the MCP server's HOME (token store); empty inherits the + // process HOME. + home string + // callbackBase is where a pasted OAuth redirect is forwarded while a + // login session is alive (the MCP server's fixed localhost listener). + callbackBase string mu sync.Mutex login *plaudLogin @@ -79,6 +85,8 @@ func newPlaudConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { command = plaudDefaultCommand } c.dial = stdioMCPDialer(command, home) + c.home = home + c.callbackBase = plaudCallbackBase return c, nil } diff --git a/src/pkg/knowledge/connector/plaud_auth_complete.go b/src/pkg/knowledge/connector/plaud_auth_complete.go new file mode 100644 index 0000000000..c129fb93bd --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth_complete.go @@ -0,0 +1,159 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// AuthCompletion carries what the operator pasted to finish a sign-in whose +// OAuth callback cannot reach the hive: either the full redirect URL the +// browser landed on, or the vendor token file contents from a laptop login. +type AuthCompletion struct { + RedirectURL string `json:"redirect_url,omitempty"` + Tokens string `json:"tokens,omitempty"` +} + +// AuthCompleter is optionally implemented by Authenticators whose OAuth +// client only allows a fixed localhost redirect (Plaud: localhost:8199). A +// hosted hive never receives that callback, so the dashboard offers two +// completions: forward the pasted redirect URL to the in-pod listener while +// the login session is alive, or install pasted tokens directly. +type AuthCompleter interface { + CompleteAuth(ctx context.Context, c AuthCompletion) (AuthState, error) +} + +var _ AuthCompleter = (*plaudConnector)(nil) + +const ( + // plaudCallbackBase is the MCP server's fixed OAuth listener. + plaudCallbackBase = "http://127.0.0.1:8199" + // plaudCallbackPath is the path the vendor registered as redirect URI. + plaudCallbackPath = "/auth/callback" + // plaudTokenFile is the MCP server's token store relative to HOME. + plaudTokenFile = ".plaud/tokens-mcp.json" + // plaudCallbackTimeout bounds the forwarded callback request. + plaudCallbackTimeout = 20 * time.Second + // plaudCompleteSettle is how long CompleteAuth waits for the login + // goroutine to notice the new session before reporting state. + plaudCompleteSettle = 6 * time.Second +) + +// ErrNoPendingLogin is returned when a redirect URL is pasted but no login +// session is alive to receive it (Connect must be called first). +var ErrNoPendingLogin = errors.New("plaud: no sign-in is pending; press Connect first, then paste the redirect URL within the login window") + +// CompleteAuth implements AuthCompleter. +func (c *plaudConnector) CompleteAuth(ctx context.Context, comp AuthCompletion) (AuthState, error) { + switch { + case strings.TrimSpace(comp.Tokens) != "": + if err := c.installTokens(comp.Tokens); err != nil { + return c.AuthStatus(ctx, false), err + } + c.mu.Lock() + if l := c.login; l != nil { + l.cancel() + c.login = nil + } + c.auth = AuthState{Pending: false, CheckedAt: c.now().UTC(), Message: "tokens installed"} + c.mu.Unlock() + return c.AuthStatus(ctx, true), nil + case strings.TrimSpace(comp.RedirectURL) != "": + return c.forwardRedirect(ctx, comp.RedirectURL) + default: + return c.AuthStatus(ctx, false), errors.New("plaud: redirect_url or tokens is required") + } +} + +// forwardRedirect replays the pasted callback against the MCP server's +// localhost listener inside the pod. +func (c *plaudConnector) forwardRedirect(ctx context.Context, raw string) (AuthState, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.RawQuery == "" { + return c.AuthStatus(ctx, false), errors.New("plaud: paste the full URL the browser landed on (it contains ?code=...&state=...)") + } + if u.Query().Get("code") == "" { + return c.AuthStatus(ctx, false), errors.New("plaud: the pasted URL has no code parameter") + } + c.mu.Lock() + l := c.login + c.mu.Unlock() + if l == nil { + return c.AuthStatus(ctx, false), ErrNoPendingLogin + } + select { + case <-l.done: + return c.AuthStatus(ctx, false), ErrNoPendingLogin + default: + } + target := strings.TrimRight(c.callbackBase, "/") + plaudCallbackPath + "?" + u.RawQuery + rctx, cancel := context.WithTimeout(ctx, plaudCallbackTimeout) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodGet, target, nil) + if err != nil { + return c.AuthStatus(ctx, false), err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return c.AuthStatus(ctx, false), fmt.Errorf("plaud: forwarding callback to the login session: %w", err) + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + if resp.StatusCode >= 400 { + return c.AuthStatus(ctx, false), fmt.Errorf("plaud: callback rejected (%s): %s", resp.Status, firstLine(string(body))) + } + // The login goroutine polls get_current_user and flips state itself; + // give it a moment so the caller sees the outcome. + settle, cancelSettle := context.WithTimeout(ctx, plaudCompleteSettle) + defer cancelSettle() + select { + case <-l.done: + case <-settle.Done(): + } + return c.AuthStatus(ctx, false), nil +} + +// installTokens validates and writes a pasted tokens-mcp.json into the +// connector's HOME so the MCP server picks it up on next start. +func (c *plaudConnector) installTokens(raw string) error { + var tok struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + } + if err := json.Unmarshal([]byte(raw), &tok); err != nil { + return fmt.Errorf("plaud: tokens must be the JSON contents of ~/.plaud/tokens-mcp.json: %w", err) + } + if tok.AccessToken == "" && tok.RefreshToken == "" { + return errors.New("plaud: tokens JSON has neither access_token nor refresh_token") + } + home := c.home + if home == "" { + h, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("plaud: resolving HOME: %w", err) + } + home = h + } + path := filepath.Join(home, plaudTokenFile) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("plaud: %w", err) + } + return os.WriteFile(path, []byte(strings.TrimSpace(raw)+"\n"), 0o600) +} + +// TokenPath returns where installTokens writes (tests). +func (c *plaudConnector) tokenPath() string { + home := c.home + if home == "" { + home, _ = os.UserHomeDir() + } + return filepath.Join(home, plaudTokenFile) +} diff --git a/src/pkg/knowledge/connector/plaud_auth_complete_test.go b/src/pkg/knowledge/connector/plaud_auth_complete_test.go new file mode 100644 index 0000000000..a4a86c5b1a --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth_complete_test.go @@ -0,0 +1,79 @@ +package connector + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestPlaudCompleteAuthInstallsPastedTokens(t *testing.T) { + srv := newPlaudTestServer() + pc := newTestPlaud(t, srv, nil) + raw := `{"access_token":"at","refresh_token":"rt","token_type":"Bearer","expires_at":1}` + st, err := pc.CompleteAuth(context.Background(), AuthCompletion{Tokens: raw}) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(pc.tokenPath()) + if err != nil { + t.Fatalf("token file not written: %v", err) + } + if strings.TrimSpace(string(data)) != raw { + t.Fatalf("token file = %q", data) + } + if !strings.HasSuffix(pc.tokenPath(), ".plaud/tokens-mcp.json") || !strings.HasPrefix(pc.tokenPath(), pc.stateDir) { + t.Fatalf("token path %q should live under the connector home", pc.tokenPath()) + } + if !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("expected refreshed connected state, got %+v", st) + } +} + +func TestPlaudCompleteAuthRejectsBadInput(t *testing.T) { + pc := newTestPlaud(t, newPlaudTestServer(), nil) + ctx := context.Background() + if _, err := pc.CompleteAuth(ctx, AuthCompletion{}); err == nil { + t.Fatal("empty completion must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{Tokens: "not json"}); err == nil { + t.Fatal("non-JSON tokens must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{Tokens: `{"token_type":"Bearer"}`}); err == nil { + t.Fatal("tokens without access/refresh token must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback"}); err == nil { + t.Fatal("redirect without query must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback?code=x&state=y"}); err != ErrNoPendingLogin { + t.Fatalf("redirect without pending login: got %v, want ErrNoPendingLogin", err) + } +} + +func TestPlaudCompleteAuthForwardsRedirectToPendingLogin(t *testing.T) { + // A live login session: the fake callback listener plays the login + // goroutine, which finishes (closes done) once the callback lands. + l := &plaudLogin{done: make(chan struct{}), cancel: func() {}} + got := make(chan string, 1) + cb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r.URL.Path + "?" + r.URL.RawQuery + close(l.done) + w.WriteHeader(http.StatusOK) + })) + defer cb.Close() + pc := newTestPlaud(t, newPlaudTestServer(), nil) + pc.callbackBase = cb.URL + pc.mu.Lock() + pc.login = l + pc.auth = AuthState{Pending: true} + pc.mu.Unlock() + _, err := pc.CompleteAuth(context.Background(), AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback?code=abc&state=xyz"}) + if err != nil { + t.Fatal(err) + } + if fwd := <-got; fwd != "/auth/callback?code=abc&state=xyz" { + t.Fatalf("forwarded %q", fwd) + } +} From 1dae54bb2cbf1505370030922679fef7ffde81d5 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 08:19:04 +0000 Subject: [PATCH 072/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.230.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.230.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-workflow-enabled-toggle.md | 1 - changelog.d/fixed-proxy-plain-forward-report.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-workflow-enabled-toggle.md delete mode 100644 changelog.d/fixed-proxy-plain-forward-report.md diff --git a/CHANGELOG.md b/CHANGELOG.md index da5445321a..fd9b669559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.230.0) + +### Added + +- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. + +### Fixed + +- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. + ## 2026-11-12 (v5.229.0) ### Added diff --git a/changelog.d/added-workflow-enabled-toggle.md b/changelog.d/added-workflow-enabled-toggle.md deleted file mode 100644 index 44fc411700..0000000000 --- a/changelog.d/added-workflow-enabled-toggle.md +++ /dev/null @@ -1 +0,0 @@ -- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. diff --git a/changelog.d/fixed-proxy-plain-forward-report.md b/changelog.d/fixed-proxy-plain-forward-report.md deleted file mode 100644 index 2e525cf0dd..0000000000 --- a/changelog.d/fixed-proxy-plain-forward-report.md +++ /dev/null @@ -1 +0,0 @@ -- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. From ea05e123db80dacae93d9c05025a0e8ca1d4cc50 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 01:43:15 -0700 Subject: [PATCH 073/223] =?UTF-8?q?=F0=9F=90=9B=20PR=20comments:=20overlap?= =?UTF-8?q?/mutation=20only=20on=20a=20finding,=20thank=20first-time=20con?= =?UTF-8?q?tributors=20only,=20no=20self-/lgtm=20relay=20on=20Prow=20PRs?= =?UTF-8?q?=20(#17870)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 PR comments: post overlap/mutation reports only on a finding, thank first-time contributors only, no self-/lgtm relay on Prow PRs Fixes #17863. - .github/scripts/pr-overlap.sh: create the <!-- hive-pr-overlap --> comment only when a sibling PR conflicts or shares a file; a later clean run edits the existing comment to a one-line Resolved note; never post for a PR that never had a finding. The step summary keeps the full report. - src/scripts/mutation-report.sh post: same for <!-- hive-mutation --> on a pass verdict; the report is appended to the step summary first. - src/scripts/feedback-gate.sh (+ test): decides whether the merge thank-you is posted — skips bot/App authors, scanner/* and hive/* branches, hive-origin bodies and returning contributors; FEEDBACK_* env knobs. Wired by the feedback.yml change in the next commit. - pkg/github: a dashboard approval on a Prow-managed, hive-authored PR relays /approve only (LGTMSkipped); Prow rejects a self-/lgtm every time and no hive lane reads the relayed /lgtm. - docs: notifications.md (new section + Prow config location), glossary, running-at-level-6; changelog fragment. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 workflows: gate the Feedback Wanted thank-you on src/scripts/feedback-gate.sh Refs #17863. Workflow-only commit: adds a gate job (self-test + decision) before the hivecommons/infra reusable-feedback call, which runs only when the gate outputs post=true. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🌱 governance: re-pin TRUSTED_SCRIPTS_TREE for the feedback-gate and mutation-report scripts Refs #17863. src/scripts gained feedback-gate.sh / test-feedback-gate.sh and mutation-report.sh changed, so the pinned tree moves to 1f1eb9048. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- .github/scripts/integration-governance.cjs | 2 +- .github/scripts/pr-overlap.sh | 27 ++++- .github/scripts/test-pr-overlap.sh | 27 ++++- .github/workflows/feedback.yml | 30 +++++ .../changed-17863-pr-comment-quartet.md | 1 + .../glossary-and-operations.md | 2 +- src/docs/notifications.md | 27 ++++- src/docs/running-at-level-6.md | 2 +- src/pkg/github/pr_decision_operator.go | 32 ++++- src/pkg/github/pr_decision_operator_test.go | 41 ++++++- src/scripts/feedback-gate.sh | 109 ++++++++++++++++++ src/scripts/mutation-report.sh | 22 ++++ src/scripts/test-feedback-gate.sh | 80 +++++++++++++ src/scripts/test-mutation-report.sh | 37 ++++++ 14 files changed, 423 insertions(+), 16 deletions(-) create mode 100644 changelog.d/changed-17863-pr-comment-quartet.md create mode 100755 src/scripts/feedback-gate.sh create mode 100755 src/scripts/test-feedback-gate.sh diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index a1cb7d505b..87ad4e2ce7 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '718b8f2888041981d0779cfd19d90484ac01b8a9'; +const TRUSTED_SCRIPTS_TREE = '1f1eb904839561af15c1772e6d9106a3b8d2b028'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/.github/scripts/pr-overlap.sh b/.github/scripts/pr-overlap.sh index f5fdaa2692..e354d26979 100755 --- a/.github/scripts/pr-overlap.sh +++ b/.github/scripts/pr-overlap.sh @@ -240,10 +240,19 @@ report_body() { } } +# One-line body the existing marker comment is edited to once a PR that had +# overlaps no longer has any. A PR that never had a finding gets no comment +# at all: the full clean report goes to the step summary only (#17863). +resolved_body() { + local base=$1 + # shellcheck disable=SC2016 # backticks are literal Markdown + printf '%s\n\n### Open PR overlap check\n\n✅ Resolved: no conflicting open PRs or same-file overlaps remain against `%s`.\n' "$MARKER" "$base" +} + upsert_comment_and_label() { - local pr=$1 body=$2 has_conflicts=$3 + local pr=$1 body=$2 has_conflicts=$3 has_any=${4:-1} base=${5:-} if ((DRY_RUN)); then - printf 'DRY RUN: would update PR #%s comment and label=%s\n' "$pr" "$has_conflicts" + printf 'DRY RUN: would update PR #%s comment and label=%s findings=%s\n' "$pr" "$has_conflicts" "$has_any" printf '%s\n' "$body" return 0 fi @@ -251,7 +260,13 @@ upsert_comment_and_label() { local comment_id comment_id=$(api "repos/${REPO}/issues/${pr}/comments?per_page=100" --paginate \ --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -n1) - if [[ -n "$comment_id" ]]; then + if [[ "$has_any" != '1' ]]; then + if [[ -n "$comment_id" ]]; then + api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -f body="$(resolved_body "$base")" >/dev/null + else + printf 'PR #%s: no overlap findings and no earlier comment; not posting\n' "$pr" + fi + elif [[ -n "$comment_id" ]]; then api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -f body="$body" >/dev/null else api -X POST "repos/${REPO}/issues/${pr}/comments" -f body="$body" >/dev/null @@ -276,12 +291,12 @@ write_summary() { run_for_one_pr() { local pr=$1 base=$2 writable=$3 truncated=${4:-0} - local rows has_conflicts _has_any base_conflict body - make_report_for_pr "$pr" "$base" rows has_conflicts _has_any base_conflict + local rows has_conflicts has_any base_conflict body + make_report_for_pr "$pr" "$base" rows has_conflicts has_any base_conflict body=$(report_body "$pr" "$base" "$rows" "$truncated" "$base_conflict") write_summary "$body" if [[ "$writable" == '1' ]]; then - upsert_comment_and_label "$pr" "$body" "$has_conflicts" + upsert_comment_and_label "$pr" "$body" "$has_conflicts" "$has_any" "$base" fi } diff --git a/.github/scripts/test-pr-overlap.sh b/.github/scripts/test-pr-overlap.sh index f3581977cc..3b553a279c 100755 --- a/.github/scripts/test-pr-overlap.sh +++ b/.github/scripts/test-pr-overlap.sh @@ -109,7 +109,7 @@ case "$*" in *'.head.repo.full_name'*) printf '%s\n' "$REPO_SLUG" ;; *'.head.sha'*) printf '%s\n' "${FALLBACK_SHA:-}" ;; *'.base.ref'*) printf 'v6\n' ;; - *'/comments?'*) : ;; + *'/comments?'*) if [[ -n "${EXISTING_COMMENT_ID:-}" ]]; then printf '%s\n' "$EXISTING_COMMENT_ID"; fi ;; *) echo "unexpected gh api call: $*" >&2; exit 2 ;; esac GH @@ -134,6 +134,8 @@ run_pr() { # pr -> sets OUT and WRITES LABEL_ADD="-X POST repos/${REPO_SLUG}/issues/%s/labels -f labels[]=conflicts-with-open-pr" LABEL_DEL="-X DELETE repos/${REPO_SLUG}/issues/%s/labels/conflicts-with-open-pr" +COMMENT_POST="-X POST repos/${REPO_SLUG}/issues/%s/comments -f body=" +COMMENT_PATCH="-X PATCH repos/${REPO_SLUG}/issues/comments/%s -f body=" echo 'PR 2: real sibling conflict with PR 3, stale PR 1 is not blamed' run_pr 2 @@ -144,6 +146,16 @@ assert_not_contains "$OUT" '| #1 |' 'does not blame PR 1 for its conflict with t assert_not_contains "$OUT" '| #4 |' 'does not link the unrelated stale PR 4' # shellcheck disable=SC2059 assert_contains "$WRITES" "$(printf -- "$LABEL_ADD" 2)" 'labels PR 2 conflicts-with-open-pr' +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_POST" 2)" 'posts the overlap comment when there is a finding' + +echo 'PR 2: an existing marker comment is edited, not duplicated' +EXISTING_COMMENT_ID=4242 run_pr 2 +assert_exit0 +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_PATCH" 4242)" 'edits the existing marker comment' +# shellcheck disable=SC2059 +assert_not_contains "$WRITES" "$(printf -- "$COMMENT_POST" 2)" 'does not post a second comment' echo 'PR 5: add/add of the same new path with different content' run_pr 5 @@ -167,9 +179,20 @@ assert_contains "$WRITES" "$(printf -- "$LABEL_DEL" 1)" 'removes the sibling-con echo 'PR 4: stale but clean, no overlap' run_pr 4 assert_exit0 -assert_contains "$OUT" 'No conflicting open PRs or same-file overlaps were found.' 'reports no overlap' +assert_contains "$OUT" 'No conflicting open PRs or same-file overlaps were found.' 'reports no overlap in the summary' # shellcheck disable=SC2059 assert_contains "$WRITES" "$(printf -- "$LABEL_DEL" 4)" 'removes the sibling-conflict label' +assert_not_contains "$WRITES" '/comments' 'posts no comment when there is nothing to report (#17863)' +assert_contains "$OUT" 'no overlap findings and no earlier comment; not posting' 'logs the skipped post' + +echo 'PR 4: a clean run edits an earlier finding comment to a one-line resolved note' +EXISTING_COMMENT_ID=777 run_pr 4 +assert_exit0 +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_PATCH" 777)<!-- hive-pr-overlap -->" 'edits the existing comment, keeping the marker' +assert_contains "$WRITES" 'Resolved: no conflicting open PRs or same-file overlaps remain against `v6`' 'collapses to the resolved line' +# shellcheck disable=SC2059 +assert_not_contains "$WRITES" "$(printf -- "$COMMENT_POST" 4)" 'does not post a new comment' echo 'PR 3: missing pull ref falls back to the API head and replaces a stale local ref' export FALLBACK_SHA diff --git a/.github/workflows/feedback.yml b/.github/workflows/feedback.yml index 0d4f5de803..14872a8e7d 100644 --- a/.github/workflows/feedback.yml +++ b/.github/workflows/feedback.yml @@ -14,5 +14,35 @@ permissions: pull-requests: write jobs: + # The reusable workflow thanks every merged PR unconditionally; the gate + # keeps the comment for a person's first merged PR only (#17863). Rules and + # knobs: src/scripts/feedback-gate.sh. + gate: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + post: ${{ steps.gate.outputs.post }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Self-test feedback gate + run: bash src/scripts/test-feedback-gate.sh + - id: gate + name: Decide whether to thank the author + env: + GH_TOKEN: ${{ github.token }} + FEEDBACK_SKIP_BOTS: '1' + FEEDBACK_SKIP_BRANCHES: 'scanner/* hive/*' + FEEDBACK_SKIP_HIVE_ORIGIN: '1' + FEEDBACK_FIRST_TIME_ONLY: '1' + run: bash src/scripts/feedback-gate.sh + feedback: + needs: gate + if: needs.gate.outputs.post == 'true' uses: hivecommons/infra/.github/workflows/reusable-feedback.yml@f1c0b61675386f54e63c04cf07b3aa933cc963eb # hivecommons/infra main as of 2026-09-02 diff --git a/changelog.d/changed-17863-pr-comment-quartet.md b/changelog.d/changed-17863-pr-comment-quartet.md new file mode 100644 index 0000000000..eb5412ae90 --- /dev/null +++ b/changelog.d/changed-17863-pr-comment-quartet.md @@ -0,0 +1 @@ +- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `<!-- hive-pr-overlap -->` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `<!-- hive-mutation -->` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 5c1d52e9ae..64a5606d00 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -225,7 +225,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str ## Non-label equivalents - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. -- **Hidden comment markers** carry state that no label does. `<!-- hive:claim ... -->` records an issue claim, `<!-- hive-finding: HASH -->` dedupes a published finding, `<!-- hive-pr-overlap -->` marks an overlap notice, `<!-- hive-question-answer -->` starts the question auto-close clock, and `<!-- hive-shared-ci-<n> -->` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#<n>` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#<n>` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#<n>` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed, timed-out or cancelled run per workflow on the PR head — using the hive's own credential, then stamps `<!-- hive-shared-ci-rerun-<n> -->` and `<!-- hive-shared-ci-lifted-<n> -->` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). +- **Hidden comment markers** carry state that no label does. `<!-- hive:claim ... -->` records an issue claim, `<!-- hive-finding: HASH -->` dedupes a published finding, `<!-- hive-pr-overlap -->` marks the one overlap notice a PR gets (posted only when a sibling PR conflicts or shares a file, edited down to a "Resolved" line when that clears; [#17863](https://github.com/hivecommons/hive/issues/17863)), `<!-- hive-question-answer -->` starts the question auto-close clock, and `<!-- hive-shared-ci-<n> -->` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#<n>` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#<n>` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#<n>` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed, timed-out or cancelled run per workflow on the PR head — using the hive's own credential, then stamps `<!-- hive-shared-ci-rerun-<n> -->` and `<!-- hive-shared-ci-lifted-<n> -->` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. - **Contributor queue holds** such as active leases, cooldowns, dependencies, and quota guard holds suppress contributor offers without touching GitHub labels. - **Jira/Linear labels** map only where the work-source adapter supports them: Linear skips held work using substring hold labels; Jira skips exact configured hold labels; GitHub Projects currently imports labels but does not use a hold gate. diff --git a/src/docs/notifications.md b/src/docs/notifications.md index d036a5c274..38427784fc 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -703,7 +703,32 @@ and exits non-zero instead of posting a possible duplicate. `.github/scripts/test-hive-status-card.sh` and `src/scripts/test-comment-merged-refs.sh` pin this behaviour. +### Per-PR workflow comments post only on a finding + +Three workflow comments used to land on every PR regardless of outcome, and +after the runtime's own reducers they were ~70% of the comments on a merged +PR ([#17863](https://github.com/hivecommons/hive/issues/17863)). Each now +posts only when it has something to say; the full report, including the +clean case, still goes to the job's step summary: + +| Workflow / script | Comment | Behaviour | +| --- | --- | --- | +| Open PR Overlap (`.github/workflows/pr-overlap.yml`, `.github/scripts/pr-overlap.sh`) | `<!-- hive-pr-overlap -->` table of conflicting / same-file open PRs | **Posts** only when at least one sibling PR conflicts or shares a file; **edits** the one marker comment in place on later pushes; when a later run finds nothing and a comment exists, **edits it** to a one-line "✅ Resolved" note; a PR that never had an overlap gets no comment. The `conflicts-with-open-pr` label is unchanged. | +| Mutation (advisory) (`.github/workflows/mutation-advisory.yml`, `src/scripts/mutation-report.sh post`) | `<!-- hive-mutation -->` surviving / uncovered mutant report | **Posts** only for a non-`pass` verdict (`lived`, `uncovered`, `misrouted`, `error`); **edits** the one marker comment on later pushes; a later `pass` **edits** the existing comment to a one-line "✅ Resolved" note; a PR whose every run passes gets no comment. | +| Feedback Wanted (`.github/workflows/feedback.yml` → `hivecommons/infra` `reusable-feedback.yml`, gated by `src/scripts/feedback-gate.sh`) | "Thank you for your contribution! Your PR has been merged." | **Posts** only for a person's first merged PR in the repo. Skipped for `[bot]` / App authors, hive-managed head branches (`scanner/*`, `hive/*`), PR bodies carrying a `<!-- hive:` marker (hive-agent PRs on any branch, e.g. `fix-*`), and any author with another merged PR. Knobs are the workflow's `env:` (`FEEDBACK_SKIP_BOTS`, `FEEDBACK_SKIP_BRANCHES`, `FEEDBACK_SKIP_HIVE_ORIGIN`, `FEEDBACK_SKIP_LOGINS`, `FEEDBACK_FIRST_TIME_ONLY`); a failed merged-PR lookup skips the comment rather than guessing. | + +`.github/scripts/test-pr-overlap.sh`, `src/scripts/test-mutation-report.sh` +and `src/scripts/test-feedback-gate.sh` pin these rules. + **Outside Hive's control.** Comments from external bots, such as Prow, tide and kubestellar-prow, are posted by their own services and cannot be edited, folded or skipped from this repository. Hive does not try to suppress them; -configure those bots in the project that runs them. +configure those bots in the project that runs them. For hivecommons repos +that configuration is `prow/plugins.yaml` and `prow/config.yaml` in +`kubestellar/infra` (org-wide `approve` with `require_self_approval: true` +and `lgtm` with `review_acts_as_lgtm: true`; tide requires both `lgtm` and +`approved`). What Hive does control is its own relay: a dashboard approval +on a Prow-managed PR the hive account opened posts `/approve` only, because +a `/lgtm` from the PR's own author draws Prow's "you cannot LGTM your own +PR" every time (`PullRequestApprovalEvent.LGTMSkipped`, #17863); no hive +lane reads the relayed `/lgtm`. diff --git a/src/docs/running-at-level-6.md b/src/docs/running-at-level-6.md index bbab0bb8f2..b7af5ce5fd 100644 --- a/src/docs/running-at-level-6.md +++ b/src/docs/running-at-level-6.md @@ -143,7 +143,7 @@ Config keys to cross-check: `project.issue_filter.require_labels`, `project.issu | `ci-flake` | CI/Hive scripts | Flake tracking, not an L6 hold itself. | Fix/quarantine the flake while keeping required checks meaningful. | | `user-feedback` | Dashboard/hub | Feedback provenance. | Triage reporter input; apply trust/hold policy as needed. | | `dco` / `no-dco` (Prow's `dco-signoff: yes|no` coexist on Prow-gated repos; the hive never adds or removes them) | Hive stage pass (commit check) | Merge-eligibility display on every managed repo. Where DCO is required (detected from required/head checks or configured via `governor.labels.stage.dco.required`), `no-dco` routes by author kind: `070` (hive/self-authored), `071` (human), or `072` (clanker); the 072 nudge includes the rebase command. On ungated repos it is informational only: the lane's stage is unchanged and no DCO rebase is requested. See [DCO routing](labels-and-control-signals/stage-labels.md#dco--no-dco--the-hive-owned-sign-off-signal). | Fix commit sign-offs (`git rebase --signoff`) rather than hand-editing labels. | -| `conflicts-with-open-pr` | PR-overlap workflow | Overlap signal. | Resolve or close conflicting work before merging; same-file overlap alone may be harmless. Two open PRs that both `Fixes` one issue are resolved by the duplicate-fix sweep without a decision: the earlier green/pending PR stays, the later hive PR is closed as `duplicate-of:<keeper>`, a person's later PR is commented on and left open ([#14799](https://github.com/hivecommons/hive/issues/14799)). | +| `conflicts-with-open-pr` | PR-overlap workflow | Overlap signal (the companion `<!-- hive-pr-overlap -->` comment appears only when there is a conflict or same-file overlap). | Resolve or close conflicting work before merging; same-file overlap alone may be harmless. Two open PRs that both `Fixes` one issue are resolved by the duplicate-fix sweep without a decision: the earlier green/pending PR stays, the later hive PR is closed as `duplicate-of:<keeper>`, a person's later PR is commented on and left open ([#14799](https://github.com/hivecommons/hive/issues/14799)). | | `has-open-pr` | Pre-dispatch guard | An open PR already links the issue with a closing keyword; the hive skipped the claim. | Nothing — the label is a tracker signal, not a park; see `scanner.skip_issues_with_open_pr` in [operator-reference.md](operator-reference.md#open-pr-dispatch-guard-and-duplicate-fix-sweep-scanner). | | `no-changelog` | Dependabot/maintainer | Repo changelog exception, not a Hive safety gate. | Apply only under repo policy. | | `human-only` remainder text (not a default label) | Agent/reviewer | Task-list sweep can park remaining issue work as `needs-human`. | Treat as a handoff; add PR `hold` if merging must stop. | diff --git a/src/pkg/github/pr_decision_operator.go b/src/pkg/github/pr_decision_operator.go index 6ca3c432df..8f0e30e6e2 100644 --- a/src/pkg/github/pr_decision_operator.go +++ b/src/pkg/github/pr_decision_operator.go @@ -94,6 +94,13 @@ type PullRequestApprovalEvent struct { // reply stays on the PR, so the next Approve takes the CommandsSkipped // path (hivecommons/hive#17475). ProwRejected bool + // LGTMSkipped is true when only `/approve` was relayed: the PR is + // Prow-managed and was opened by the hive's own account, so a `/lgtm` + // posted from that account is the PR author's and Prow's lgtm plugin + // answers every one with "you cannot LGTM your own PR" + // (hivecommons/hive#17863). No hive lane reads the relayed `/lgtm`; + // the `/approve` comment carries the attribution they use. + LGTMSkipped bool // ReportingError preserves partial progress: the native approval stands, // but comments, labels or stage reporting must be retried/reconciled. ReportingError string @@ -503,19 +510,40 @@ func (c *Client) reportOperatorPullRequestApproval(ctx context.Context, owner, n c.logger.Info("pull request approval commands not posted: Prow rejected the hive's earlier commands on this PR (bot is not an approver); the hive's own approval record stands", "repo", event.Repo, "pr", number, "actor", actor, "head", head) } } else { - for _, command := range PRApprovalCommands { + commands := PRApprovalCommands + if event.ProwManaged && selfAuthored { + commands = prApprovalCommandsWithoutLGTM(commands) + event.LGTMSkipped = true + } + // Each relayed command carries the prApprovalCommentMarkerPrefix + // marker bound to head, which is what prowRejectedHiveCommands and + // the sentinel/workflow-approval readers key on; once posted for a + // head it is not posted again (CommandsSkipped above). + for _, command := range commands { if err := c.CreateIssueComment(ctx, full, number, renderPRApprovalCommandComment(command, actor, head)); err != nil { return fmt.Errorf("posting %s on %s/%s#%d: %w", command, owner, name, number, err) } } if c.logger != nil { - c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head, "lgtm_skipped_self_authored", event.LGTMSkipped) } event.ProwRejected = c.awaitProwRejection(ctx, owner, name, number, event.ProwManaged) } return ctx.Err() } +// prApprovalCommandsWithoutLGTM returns commands minus `/lgtm`, for a +// Prow-managed PR the posting account itself authored (#17863). +func prApprovalCommandsWithoutLGTM(commands []string) []string { + out := make([]string, 0, len(commands)) + for _, command := range commands { + if command != "/lgtm" { + out = append(out, command) + } + } + return out +} + // ProwManagedLabels reports whether labels carry Prow's plugin markers: // the approve plugin's `approved`, size's `size/*`, the `do-not-merge/*` // family and trigger's `ok-to-test` / `needs-ok-to-test`. diff --git a/src/pkg/github/pr_decision_operator_test.go b/src/pkg/github/pr_decision_operator_test.go index c6a5746ad2..96afbdc68a 100644 --- a/src/pkg/github/pr_decision_operator_test.go +++ b/src/pkg/github/pr_decision_operator_test.go @@ -390,6 +390,42 @@ func TestRecordOperatorPullRequestApprovalSkipsCommandsProwRejected(t *testing.T } } +// On a Prow-managed PR the hive account opened, a `/lgtm` from that account +// is a self-LGTM Prow always rebukes; only `/approve` is relayed +// (hivecommons/hive#17863). A person's PR and a non-Prow repo keep both. +func TestRecordOperatorPullRequestApprovalSkipsSelfLGTMOnProwPR(t *testing.T) { + for _, tc := range []struct { + name string + pr unparkWireIssue + wantLGTM bool + wantSkipped bool + }{ + {"hive-authored prow-managed", parkedPR(liveDecisionPR, "hive[bot]", []string{PRNeedsDecisionStageLabel, "author/hive", "size/M"}), false, true}, + {"hive-authored not prow-managed", hiveParkedPR(), true, false}, + {"person-authored prow-managed", parkedPR(liveDecisionPR, "contributor", []string{PRNeedsDecisionStageLabel, "author/human", "size/M"}), true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + c, rec := newUnparkPRClient(t, []unparkWireIssue{tc.pr}, map[string]string{"maintainer": "write"}) + event, err := c.RecordOperatorPullRequestApproval(context.Background(), "org/repo", liveDecisionPR, "maintainer") + if err != nil { + t.Fatalf("approval: %v", err) + } + if event.LGTMSkipped != tc.wantSkipped { + t.Fatalf("event = %+v, want LGTMSkipped=%v", event, tc.wantSkipped) + } + posted := rec.posted[liveDecisionPR] + var gotLGTM, gotApprove bool + for _, body := range posted { + gotLGTM = gotLGTM || strings.HasPrefix(body, "/lgtm\n") + gotApprove = gotApprove || strings.HasPrefix(body, "/approve\n") + } + if gotLGTM != tc.wantLGTM || !gotApprove { + t.Fatalf("posted %v, want lgtm=%v and /approve", posted, tc.wantLGTM) + } + }) + } +} + // On a Prow repo where rejection has not been observed yet, Prow answers the // relayed commands after they are posted. The approval event reports that // reply instead of leaving the board at "approved" (hivecommons/hive#17475). @@ -418,8 +454,9 @@ func TestRecordOperatorPullRequestApprovalReportsProwRejectionAfterPost(t *testi if err != nil { t.Fatalf("approval: %v", err) } - if event.ProwRejected != tc.want || event.CommandsSkipped != "" || len(rec.posted[liveDecisionPR]) != len(PRApprovalCommands) { - t.Fatalf("event = %+v, want ProwRejected %v with commands posted", event, tc.want) + // Hive-authored and Prow-managed: only `/approve` is relayed (#17863). + if event.ProwRejected != tc.want || event.CommandsSkipped != "" || !event.LGTMSkipped || len(rec.posted[liveDecisionPR]) != len(PRApprovalCommands)-1 { + t.Fatalf("event = %+v, want ProwRejected %v with /approve posted", event, tc.want) } }) } diff --git a/src/scripts/feedback-gate.sh b/src/scripts/feedback-gate.sh new file mode 100755 index 0000000000..50c1c0e314 --- /dev/null +++ b/src/scripts/feedback-gate.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# feedback-gate.sh — decides whether the post-merge "Thank you for your +# contribution!" comment (.github/workflows/feedback.yml → +# hivecommons/infra reusable-feedback.yml) should be posted on a merged PR. +# +# The reusable workflow is unconditional, so before #17863 every merged PR +# got the comment: 38 of 40 in the 2026-11-11 spam sample were on hive-App or +# maintainer PRs that nobody reads it on. The comment is for people merging +# their first change, so the gate keeps it for exactly that case. +# +# Reads the pull_request event from $GITHUB_EVENT_PATH (or the FEEDBACK_PR_* +# overrides below, used by the self-test) and writes `post=true|false` and +# `reason=<why>` to $GITHUB_OUTPUT (stdout when unset). Exit status is 0 in +# both cases; only a broken event or API read exits non-zero, which the caller +# treats as "do not post". +# +# Configuration (workflow `env:`), each with its default: +# FEEDBACK_SKIP_BOTS=1 skip `[bot]` / type=Bot authors +# FEEDBACK_SKIP_BRANCHES='scanner/* hive/*' +# space-separated globs of hive-managed +# head branches to skip +# FEEDBACK_SKIP_HIVE_ORIGIN=1 skip PRs whose body carries a `<!-- hive:` +# marker (hive-agent PRs on any branch, +# e.g. `fix-*`) +# FEEDBACK_SKIP_LOGINS='' space-separated logins never thanked +# FEEDBACK_FIRST_TIME_ONLY=1 skip authors with another merged PR in +# the repo (needs GH_TOKEN; a failed lookup +# skips the comment rather than guessing) +# +# Overrides for tests / manual runs: FEEDBACK_PR_AUTHOR, FEEDBACK_PR_AUTHOR_TYPE, +# FEEDBACK_PR_HEAD_REF, FEEDBACK_PR_BODY, FEEDBACK_PR_MERGED, FEEDBACK_PR_NUMBER, +# FEEDBACK_MERGED_COUNT (skips the API lookup). +set -euo pipefail + +skip_bots="${FEEDBACK_SKIP_BOTS:-1}" +skip_branches="${FEEDBACK_SKIP_BRANCHES-scanner/* hive/*}" +skip_hive_origin="${FEEDBACK_SKIP_HIVE_ORIGIN:-1}" +skip_logins="${FEEDBACK_SKIP_LOGINS:-}" +first_time_only="${FEEDBACK_FIRST_TIME_ONLY:-1}" +repo="${GITHUB_REPOSITORY:-}" + +lower() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } + +event_field() { + local path=$1 + if [[ -n "${GITHUB_EVENT_PATH:-}" && -f "${GITHUB_EVENT_PATH}" ]]; then + jq -r "${path} // empty" "$GITHUB_EVENT_PATH" + fi +} + +author="${FEEDBACK_PR_AUTHOR:-$(event_field '.pull_request.user.login')}" +author_type="${FEEDBACK_PR_AUTHOR_TYPE:-$(event_field '.pull_request.user.type')}" +head_ref="${FEEDBACK_PR_HEAD_REF:-$(event_field '.pull_request.head.ref')}" +body="${FEEDBACK_PR_BODY-$(event_field '.pull_request.body')}" +merged="${FEEDBACK_PR_MERGED:-$(event_field '.pull_request.merged')}" +number="${FEEDBACK_PR_NUMBER:-$(event_field '.pull_request.number')}" + +decide() { + local post=$1 reason=$2 + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf 'post=%s\nreason=%s\n' "$post" "$reason" >> "$GITHUB_OUTPUT" + fi + printf 'feedback-gate: post=%s (%s)\n' "$post" "$reason" + exit 0 +} + +[[ -n "$author" ]] || { echo 'feedback-gate: no PR author in the event' >&2; exit 1; } +[[ "$merged" == 'true' ]] || decide false 'pr-not-merged' + +if [[ "$skip_bots" == '1' ]] && { [[ "$author" == *'[bot]' ]] || [[ "$(lower "$author_type")" == 'bot' ]]; }; then + decide false "bot-author:${author}" +fi + +for login in $skip_logins; do + if [[ "$(lower "$login")" == "$(lower "$author")" ]]; then + decide false "listed-login:${author}" + fi +done + +for pattern in $skip_branches; do + # shellcheck disable=SC2053 # the glob in $pattern is meant to expand + if [[ -n "$head_ref" && "$head_ref" == $pattern ]]; then + decide false "hive-branch:${head_ref}" + fi +done + +if [[ "$skip_hive_origin" == '1' && "$body" == *'<!-- hive:'* ]]; then + decide false 'hive-origin-body' +fi + +if [[ "$first_time_only" == '1' ]]; then + merged_count="${FEEDBACK_MERGED_COUNT:-}" + if [[ -z "$merged_count" ]]; then + [[ -n "$repo" ]] || { echo 'feedback-gate: GITHUB_REPOSITORY unset' >&2; exit 1; } + # The event fires after the merge, so the PR being judged is already one + # of the author's merged PRs: "first-time" means it is the only one. + merged_count=$(gh api -X GET search/issues -f "q=repo:${repo} is:pr is:merged author:${author}" -F per_page=1 --jq '.total_count') \ + || { echo "feedback-gate: merged-PR lookup for ${author} failed; not posting" >&2; exit 1; } + fi + if [[ ! "$merged_count" =~ ^[0-9]+$ ]]; then + echo "feedback-gate: unexpected merged count '${merged_count}'" >&2 + exit 1 + fi + if (( merged_count > 1 )); then + decide false "returning-contributor:${author}:${merged_count}-merged" + fi +fi + +decide true "first-merged-pr:${author}${number:+:#${number}}" diff --git a/src/scripts/mutation-report.sh b/src/scripts/mutation-report.sh index 6bac832ad0..8dcf9c2522 100755 --- a/src/scripts/mutation-report.sh +++ b/src/scripts/mutation-report.sh @@ -100,11 +100,33 @@ post_comment() { echo "PR_NUMBER/GITHUB_REPOSITORY unset; not posting a comment." return 0 fi + # The report always lands in the step summary; the PR comment is reserved + # for findings (any verdict but `pass`). A clean run never creates a + # comment, and one that follows an earlier finding edits that comment down + # to a one-line resolved note instead of posting again (#17863). + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$body" >> "$GITHUB_STEP_SUMMARY" + fi + local verdict="" resolved="" + [ -f "$verdict_file" ] && verdict=$(cat "$verdict_file") + if [ "$verdict" = pass ]; then + # shellcheck disable=SC2016 # backticks are literal Markdown + resolved="$(printf '%s\n### Mutation testing\n\n✅ Resolved: no surviving or uncovered mutants in changed files on the current head.\n\n_Verdict: `pass` (#17349)._\n' "$marker")" + fi # Only upsert the bot's own report so another participant cannot pre-seat or # get overwritten by a comment that merely starts with the marker. local id id=$(gh api --paginate "repos/${repo}/issues/${pr_number}/comments" \ --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"${marker}\"))) | .id" 2>/dev/null | head -n 1 || true) + if [ -n "$resolved" ]; then + if [ -z "$id" ]; then + echo "mutation: verdict pass and no earlier comment; not posting (report is in the step summary)." + return 0 + fi + gh api -X PATCH "repos/${repo}/issues/comments/${id}" -f "body=${resolved}" >/dev/null \ + || echo "::warning::could not edit mutation comment" + return 0 + fi if [ -n "$id" ]; then gh api -X PATCH "repos/${repo}/issues/comments/${id}" -F "body=@${body}" >/dev/null \ || echo "::warning::could not edit mutation comment" diff --git a/src/scripts/test-feedback-gate.sh b/src/scripts/test-feedback-gate.sh new file mode 100755 index 0000000000..3864f8ff04 --- /dev/null +++ b/src/scripts/test-feedback-gate.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# test-feedback-gate.sh — pins the gating rules of feedback-gate.sh (#17863) +# with a stub `gh` so no network access is needed. +set -u -o pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="${HERE}/feedback-gate.sh" +TMP_ROOT="${HERE}/../.test-tmp" +mkdir -p "$TMP_ROOT" +TMP="$(mktemp -d "$TMP_ROOT/feedback-gate.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +fail=0 +pass() { echo " ok: $*"; } +bad() { echo " FAIL: $*"; fail=1; } + +mkdir -p "$TMP/bin" +cat > "$TMP/bin/gh" <<'GHSTUB' +#!/usr/bin/env bash +case "$*" in + *search/issues*) + [ -n "${GH_FAIL:-}" ] && exit 1 + printf '%s\n' "${GH_TOTAL:-1}" ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +GHSTUB +chmod +x "$TMP/bin/gh" + +# run <expected-post> <label> [VAR=value ...] +run() { + local want=$1 label=$2; shift 2 + local out="$TMP/output"; : > "$out" + local result rc + result=$(env -i PATH="$TMP/bin:/usr/bin:/bin" HOME="$TMP" GITHUB_OUTPUT="$out" GITHUB_REPOSITORY=example/hive \ + FEEDBACK_PR_MERGED=true FEEDBACK_PR_AUTHOR=newcomer FEEDBACK_PR_AUTHOR_TYPE=User \ + FEEDBACK_PR_HEAD_REF=feat/thing FEEDBACK_PR_BODY='plain body' FEEDBACK_PR_NUMBER=7 \ + "$@" bash "$SCRIPT" 2>&1); rc=$? + if [ "$want" = error ]; then + [ "$rc" -ne 0 ] && ! grep -q '^post=' "$out" && pass "$label (exit $rc, no output)" || bad "$label: rc=$rc out=$(cat "$out") $result" + return + fi + if [ "$rc" -eq 0 ] && grep -qx "post=$want" "$out"; then pass "$label ($(grep '^reason=' "$out"))"; else bad "$label: rc=$rc out=$(cat "$out") $result"; fi +} + +echo "default rules" +run true 'human first merged PR is thanked' GH_TOTAL=1 +run false 'returning contributor is not thanked' GH_TOTAL=3 +run false 'App author is skipped' FEEDBACK_PR_AUTHOR='hivecommons-hive[bot]' FEEDBACK_PR_AUTHOR_TYPE=Bot +run false 'type=Bot without [bot] suffix is skipped' FEEDBACK_PR_AUTHOR=somebot FEEDBACK_PR_AUTHOR_TYPE=Bot +run false 'scanner/* branch is skipped' FEEDBACK_PR_HEAD_REF=scanner/v6-claude-auth +run false 'hive/* branch is skipped' FEEDBACK_PR_HEAD_REF=hive/adopt-123 +run false 'hive-origin body is skipped on any branch' FEEDBACK_PR_HEAD_REF=fix-17863-feedback FEEDBACK_PR_BODY=$'Fixes #1\n<!-- hive:origin issue=1 -->' +run true 'fix-* branch by a person is still thanked' FEEDBACK_PR_HEAD_REF=fix-typo GH_TOTAL=1 +run false 'unmerged (closed) PR is never thanked' FEEDBACK_PR_MERGED=false +run error 'merged-PR lookup failure exits non-zero without an output' GH_FAIL=1 +run error 'garbage merged count exits non-zero' GH_TOTAL=lots + +echo "configuration" +run true 'FEEDBACK_SKIP_BOTS=0 thanks a bot' FEEDBACK_SKIP_BOTS=0 FEEDBACK_PR_AUTHOR='x[bot]' FEEDBACK_PR_AUTHOR_TYPE=Bot GH_TOTAL=1 +run true 'FEEDBACK_FIRST_TIME_ONLY=0 thanks a returning contributor' FEEDBACK_FIRST_TIME_ONLY=0 GH_TOTAL=9 +run false 'FEEDBACK_SKIP_BRANCHES adds a pattern' FEEDBACK_SKIP_BRANCHES='scanner/* release/*' FEEDBACK_PR_HEAD_REF=release/v5.1 +run true 'FEEDBACK_SKIP_BRANCHES= disables branch skipping' FEEDBACK_SKIP_BRANCHES= FEEDBACK_PR_HEAD_REF=scanner/x GH_TOTAL=1 +run false 'FEEDBACK_SKIP_LOGINS is case-insensitive' FEEDBACK_SKIP_LOGINS='Maintainer other' FEEDBACK_PR_AUTHOR=maintainer +run true 'FEEDBACK_MERGED_COUNT override skips the API' FEEDBACK_MERGED_COUNT=1 GH_FAIL=1 + +echo "event file" +cat > "$TMP/event.json" <<'EOF' +{"pull_request":{"number":17716,"merged":true,"user":{"login":"clubanderson","type":"User"},"head":{"ref":"fix/thing"},"body":"body"}} +EOF +out="$TMP/output"; : > "$out" +result=$(env -i PATH="$TMP/bin:/usr/bin:/bin" HOME="$TMP" GITHUB_OUTPUT="$out" GITHUB_REPOSITORY=example/hive \ + GITHUB_EVENT_PATH="$TMP/event.json" GH_TOTAL=400 bash "$SCRIPT" 2>&1); rc=$? +[ "$rc" -eq 0 ] && grep -qx 'post=false' "$out" && grep -q 'returning-contributor:clubanderson:400-merged' "$out" \ + && pass "fields are read from GITHUB_EVENT_PATH" || bad "event file: rc=$rc out=$(cat "$out") $result" + +if [ "$fail" -ne 0 ]; then + echo "test-feedback-gate.sh: FAILED" + exit 1 +fi +echo "test-feedback-gate.sh: all checks passed" diff --git a/src/scripts/test-mutation-report.sh b/src/scripts/test-mutation-report.sh index 3da43953bb..f41b6a9bdf 100755 --- a/src/scripts/test-mutation-report.sh +++ b/src/scripts/test-mutation-report.sh @@ -200,6 +200,43 @@ output=$(MUTATION_OUT_DIR="$TMP/missing" bash "$SCRIPT" verdict 2>&1); rc=$? [ "$rc" -eq 1 ] && out_grep -q '::error::mutation: no verdict at' \ && pass "missing verdict file fails the verdict phase" || bad "missing verdict rc=$rc: $output" +echo "post phase (#17863)" +# Stub gh: lists $GH_EXISTING_ID (if set) as the bot's marker comment and logs writes. +cat > "$TMP/bin/gh" <<'GHSTUB' +#!/usr/bin/env bash +case "$*" in + *-X*) printf '%s\n' "$*" >> "$GH_WRITE_LOG"; exit 0 ;; + *'/comments'*) if [ -n "${GH_EXISTING_ID:-}" ]; then printf '%s\n' "$GH_EXISTING_ID"; fi; exit 0 ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +GHSTUB +chmod +x "$TMP/bin/gh" +run_post() { # run_post <out-dir> [existing-id] + GH_WRITE_LOG="$TMP/gh-writes.log"; : > "$GH_WRITE_LOG" + SUMMARY="$TMP/summary.md"; : > "$SUMMARY" + output=$(cd "$repo" && GH_WRITE_LOG="$GH_WRITE_LOG" GH_EXISTING_ID="${2:-}" GITHUB_STEP_SUMMARY="$SUMMARY" \ + MUTATION_OUT_DIR="$1" PR_NUMBER=42 GITHUB_REPOSITORY=example/hive bash "$SCRIPT" post 2>&1) + writes=$(cat "$GH_WRITE_LOG") +} +mkdir -p "$TMP/out-pass" "$TMP/out-lived" +printf '<!-- hive-mutation -->\n### Mutation testing\n\nAll 3 mutant(s) in changed files were killed.\n' > "$TMP/out-pass/comment.md" +echo pass > "$TMP/out-pass/verdict" +printf '<!-- hive-mutation -->\n### Mutation testing\n\ngremlins found **1 surviving mutant(s)**\n' > "$TMP/out-lived/comment.md" +echo lived > "$TMP/out-lived/verdict" +run_post "$TMP/out-pass" +[ -z "$writes" ] && pass "pass verdict with no earlier comment posts nothing" || bad "pass posted: $writes" +out_grep -q 'verdict pass and no earlier comment; not posting' && pass "skip is logged" || bad "skip not logged: $output" +grep -q 'All 3 mutant(s) in changed files were killed' "$SUMMARY" && pass "clean report still lands in the step summary" || bad "summary missing report" +run_post "$TMP/out-lived" +grep -q "^api -X POST repos/example/hive/issues/42/comments" <<<"$writes" && pass "lived verdict posts a new comment" || bad "lived did not post: $writes" +run_post "$TMP/out-lived" 99 +grep -q "^api -X PATCH repos/example/hive/issues/comments/99" <<<"$writes" && pass "lived verdict edits the existing comment" || bad "lived did not edit: $writes" +grep -q 'POST' <<<"$writes" && bad "lived posted a duplicate" || pass "no duplicate comment" +run_post "$TMP/out-pass" 99 +grep -q "^api -X PATCH repos/example/hive/issues/comments/99 -f body=<!-- hive-mutation -->" <<<"$writes" && pass "pass after a finding edits the existing comment, keeping the marker" || bad "pass did not edit: $writes" +grep -q 'Resolved: no surviving or uncovered mutants' <<<"$writes" && pass "edited body is the one-line resolved note" || bad "resolved note missing: $writes" +grep -q 'All 3 mutant' <<<"$writes" && bad "full report was written to the comment" || pass "full report stays out of the comment" + if [ "$fail" -ne 0 ]; then echo "test-mutation-report.sh: FAILED" exit 1 From 05b4c6c1965516c49d96efd46a10aad732fc4fd0 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 01:46:33 -0700 Subject: [PATCH 074/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20atomic=20runtime?= =?UTF-8?q?=20config=20save=20+=20fresh=20legacy=20fallback=20(#17862)=20(?= =?UTF-8?q?#17867)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: atomic runtime config save + fresh legacy fallback (#17862) Config.Save wrote /etc/hive/hive.yaml and /data/hive.yaml.runtime in place with O_TRUNC. A liveness SIGKILL between truncate and write left a 0-byte runtime file; the entrypoint's -s check then fell back to /data/hive.yaml.bak, frozen at the Aug 2026 rename on every hive, and vllmd-03 booted level 3 instead of 6. - writeFileAtomic: temp file in the target's dir, fsync, rename onto the EvalSymlinks-resolved target so the #17468 link survives; falls back to the old in-place write when rename is refused (single-file bind mounts). - Both Save writes use it; runtime stays 0600, source keeps its mode. - entrypoint: hive_refresh_legacy_fallback copies a YAML-valid runtime config over hive.yaml.bak on clean boots (tmp+mv), only when the new name is the boot source; hive_runtime_config_read WARNs loudly on fallback and flags a fallback older than the damaged runtime file. - Tests: writer unit tests incl. symlink + no-partial-read race; shell suite covers WARN, refresh, parse guard, legacy-boot no-op. Fixes #17862 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * test(config): cover atomic writer failure paths for coverage floor Adds tests for missing target dir, ENOTDIR resolve error, dangling symlink, rename+fallback double failure (no temp leak), in-place fallback, syncDir tolerance and sourceConfigMode, lifting pkg/config back over its 90% coverage floor. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- .../fixed-17862-atomic-runtime-config-save.md | 1 + src/deploy/entrypoint.sh | 77 ++++- src/deploy/test_entrypoint_runtime_config.sh | 112 ++++++- src/pkg/config/atomic_write.go | 116 +++++++ src/pkg/config/atomic_write_test.go | 288 ++++++++++++++++++ src/pkg/config/config.go | 70 ++--- src/pkg/hub/saas_provision.go | 11 +- 7 files changed, 612 insertions(+), 63 deletions(-) create mode 100644 changelog.d/fixed-17862-atomic-runtime-config-save.md create mode 100644 src/pkg/config/atomic_write.go create mode 100644 src/pkg/config/atomic_write_test.go diff --git a/changelog.d/fixed-17862-atomic-runtime-config-save.md b/changelog.d/fixed-17862-atomic-runtime-config-save.md new file mode 100644 index 0000000000..0dad10650e --- /dev/null +++ b/changelog.d/fixed-17862-atomic-runtime-config-save.md @@ -0,0 +1 @@ +- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index 66e68db376..87e19e6667 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -64,7 +64,9 @@ EXIT_NET_ADMIN_REQUIRED=77 # file is the only copy of the live config, and mutating it at boot risks # losing owner customisations with no warning. The old name is picked up # read-only until the first save writes the new one, after which both -# exist and the new one wins. +# exist and the new one wins. Once the new name is the boot source, every +# clean boot refreshes the old name from it (#17862) so the fallback is +# never months stale; see hive_refresh_legacy_fallback. HIVE_CONFIG_PATH="${HIVE_CONFIG:-/etc/hive/hive.yaml}" HIVE_CONFIG_RUNTIME="/data/hive.yaml.runtime" HIVE_CONFIG_RUNTIME_LEGACY="/data/hive.yaml.bak" @@ -771,18 +773,89 @@ for _cfg in "$HIVE_CONFIG_RUNTIME" "$HIVE_CONFIG_RUNTIME_LEGACY" /data/hive.yaml done unset _cfg +# hive_file_mtime echoes a file's mtime as epoch seconds (0 when unreadable). +# Both stat dialects are tried because the image is Debian but the helper +# must also behave on the BusyBox/macOS shells the tests run under. +hive_file_mtime() { + stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0 +} + # hive_runtime_config_read echoes the path to read the persisted runtime # config from: the new name when it is present and non-empty, else the # legacy name when that is, else empty. Read-only — it never creates, # renames or removes anything on the PVC. +# +# #17862: falling back to the legacy name is never routine any more. Every +# clean boot refreshes it from the runtime config (hive_refresh_legacy_fallback), +# so reaching it means the runtime config was lost or truncated since the +# last boot. Say so LOUDLY on stderr with both files' ages: a hive that +# silently booted a 3-month-old hive.yaml.bak (level 3 instead of 6) is the +# incident this guards against. When the legacy copy is OLDER than the +# runtime file it is replacing — i.e. the runtime file was written after +# the last refresh and then damaged — the WARN says the fallback is stale +# and names the overlay/audit paths an operator can reconstruct from. hive_runtime_config_read() { if [ -f "$HIVE_CONFIG_RUNTIME" ] && [ -s "$HIVE_CONFIG_RUNTIME" ]; then echo "$HIVE_CONFIG_RUNTIME" elif [ -f "$HIVE_CONFIG_RUNTIME_LEGACY" ] && [ -s "$HIVE_CONFIG_RUNTIME_LEGACY" ]; then + _rt_state="missing" + _rt_mtime=0 + if [ -f "$HIVE_CONFIG_RUNTIME" ]; then + _rt_state="present but EMPTY (0 bytes — likely truncated by a kill mid-save)" + _rt_mtime="$(hive_file_mtime "$HIVE_CONFIG_RUNTIME")" + fi + _lg_mtime="$(hive_file_mtime "$HIVE_CONFIG_RUNTIME_LEGACY")" + _now="$(date +%s)" + _lg_age_h=$(( (_now - _lg_mtime) / 3600 )) + echo "[entrypoint] WARN: runtime config $HIVE_CONFIG_RUNTIME is $_rt_state; FALLING BACK to legacy $HIVE_CONFIG_RUNTIME_LEGACY (last refreshed ${_lg_age_h}h ago)." >&2 + if [ "$_rt_mtime" -gt "$_lg_mtime" ] 2>/dev/null; then + echo "[entrypoint] WARN: the legacy fallback is OLDER than the damaged runtime config ($(( (_rt_mtime - _lg_mtime) / 3600 ))h behind). Settings saved since then (acmm_level, agent roster, pauses, models) are NOT in this file — check /data/hive.yaml.dashboard and /data/audit.jsonl before trusting the booted config. See hivecommons/hive#17862." >&2 + fi + unset _rt_state _rt_mtime _lg_mtime _now _lg_age_h echo "$HIVE_CONFIG_RUNTIME_LEGACY" fi } +# hive_refresh_legacy_fallback copies a GOOD runtime config over the legacy +# fallback name so the fallback is at most one boot stale (#17862). Before +# this, hive.yaml.bak was frozen at the rename (Aug 2026) on every hive, and +# a truncated runtime file booted a months-old config. +# +# Only runs when THIS boot is reading from the new name (so the legacy file +# is strictly older, never the only copy — the Docker/LXC worry behind the +# original never-write rule) and the runtime file parses as YAML with a +# project section (so a half-written file is never promoted to fallback). +# Copies to a temp name and renames so the fallback is never itself +# truncated by a kill. Best-effort: a failure here only means the fallback +# stays at its previous age. +hive_refresh_legacy_fallback() { + [ "$1" = "$HIVE_CONFIG_RUNTIME" ] || return 0 + [ -s "$HIVE_CONFIG_RUNTIME" ] || return 0 + if command -v python3 >/dev/null 2>&1; then + python3 -c ' +import sys, yaml +with open(sys.argv[1]) as f: + d = yaml.safe_load(f) +sys.exit(0 if isinstance(d, dict) and isinstance(d.get("project"), dict) else 1) +' "$HIVE_CONFIG_RUNTIME" 2>/dev/null || { + echo "[entrypoint] WARN: $HIVE_CONFIG_RUNTIME does not parse as a hive config; legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY left untouched" >&2 + return 0 + } + fi + if cmp -s "$HIVE_CONFIG_RUNTIME" "$HIVE_CONFIG_RUNTIME_LEGACY" 2>/dev/null; then + return 0 + fi + _lg_tmp="${HIVE_CONFIG_RUNTIME_LEGACY}.tmp.$$" + if cp "$HIVE_CONFIG_RUNTIME" "$_lg_tmp" 2>/dev/null && mv -f "$_lg_tmp" "$HIVE_CONFIG_RUNTIME_LEGACY" 2>/dev/null; then + hive_harden_runtime_config "$HIVE_CONFIG_RUNTIME_LEGACY" + echo "[entrypoint] Legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY refreshed from $HIVE_CONFIG_RUNTIME" + else + rm -f "$_lg_tmp" 2>/dev/null + echo "[entrypoint] WARN: could not refresh legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY from $HIVE_CONFIG_RUNTIME" >&2 + fi + unset _lg_tmp +} + # hive_ghe_git_host echoes this hive's configured GitHub host when it is NOT # public github.com (i.e. a GitHub Enterprise instance such as github.ibm.com), # and nothing at all otherwise. Derived the same way the Go binary's @@ -1018,6 +1091,7 @@ if [ "$IS_KUBERNETES" = "true" ]; then # result equal to its own input on every boot. HIVE_CONFIG_BOOT="$(hive_runtime_config_read)" if [ -n "$HIVE_CONFIG_BOOT" ]; then + hive_refresh_legacy_fallback "$HIVE_CONFIG_BOOT" # Steady state: boot from what this hive last had. The ConfigMap seed is # deliberately NOT merged — it is frozen at provision and every field it # could contribute is either already here or re-asserted by the hub on @@ -1207,6 +1281,7 @@ else # first boot on new code seeds hive.yaml.runtime from the legacy content # without touching the legacy file itself. HIVE_CONFIG_SOURCE="$(hive_runtime_config_read)" + hive_refresh_legacy_fallback "$HIVE_CONFIG_SOURCE" if [ -f "$HIVE_CONFIG_PATH" ] && [ -s "$HIVE_CONFIG_PATH" ] && [ -z "$HIVE_CONFIG_SOURCE" ]; then # First boot: config exists but no PVC runtime config yet — seed it cp "$HIVE_CONFIG_PATH" "$HIVE_CONFIG_RUNTIME" diff --git a/src/deploy/test_entrypoint_runtime_config.sh b/src/deploy/test_entrypoint_runtime_config.sh index 4f97399d2c..6e21a20570 100755 --- a/src/deploy/test_entrypoint_runtime_config.sh +++ b/src/deploy/test_entrypoint_runtime_config.sh @@ -2,10 +2,13 @@ # Tests the hive.yaml.bak -> hive.yaml.runtime migration in entrypoint.sh. # # The rename is copy-forward: readers prefer /data/hive.yaml.runtime and fall -# back to the legacy /data/hive.yaml.bak, and NOTHING renames or removes the -# legacy file on the PVC. ~51 live hives carry only the legacy name, and on -# Docker/LXC that file is the single copy of the live config — mutating it at -# boot could lose owner customisations with no warning. +# back to the legacy /data/hive.yaml.bak, and NOTHING removes the legacy file +# on the PVC. ~51 live hives carry only the legacy name, and on Docker/LXC +# that file is the single copy of the live config. +# +# #17862 added one deliberate write: when the NEW name is the boot source and +# parses as a config, the legacy file is refreshed from it (temp + mv) so the +# fallback is never months stale, and falling back to it logs a loud WARN. # # Rather than grepping the script, this extracts the real resolver function and # executes it against real files, so it fails if the logic regresses. @@ -38,9 +41,9 @@ echo "=== entrypoint runtime-config migration tests ===" # Extract the resolver verbatim from the entrypoint so this tests the shipped # code, not a copy that can drift away from it. -RESOLVER="$(sed -n '/^hive_runtime_config_read() {/,/^}/p' "$ENTRYPOINT")" -if [ -z "$RESOLVER" ]; then - echo " FAIL: could not extract hive_runtime_config_read from $ENTRYPOINT" +RESOLVER="$(sed -n '/^hive_file_mtime() {/,/^}/p; /^hive_runtime_config_read() {/,/^}/p' "$ENTRYPOINT")" +if ! grep -q 'hive_runtime_config_read() {' <<<"$RESOLVER" || ! grep -q 'hive_file_mtime() {' <<<"$RESOLVER"; then + echo " FAIL: could not extract hive_runtime_config_read / hive_file_mtime from $ENTRYPOINT" exit 1 fi @@ -63,7 +66,7 @@ resolve() { export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY local out - out="$(eval "$RESOLVER"; hive_runtime_config_read)" + out="$(eval "$RESOLVER"; hive_runtime_config_read 2>/dev/null)" case "$out" in "$HIVE_CONFIG_RUNTIME") echo "new" ;; @@ -95,17 +98,96 @@ check "empty new + good legacy -> legacy" "legacy" "$(resolve "EMPTY" "old: cfg" # 6. Both empty is the same as neither. check "empty new + empty legacy -> empty" "none" "$(resolve "EMPTY" "EMPTY")" -# 7. The legacy file must never be renamed or deleted. On Docker/LXC it is the -# only copy of the live config, so a destructive migration could lose owner -# customisations silently. -if grep -Eq '(mv|rm)[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT"; then - echo " FAIL: entrypoint moves or removes the legacy runtime config" - echo " the migration must be copy-forward only" +# 7. The legacy file must never be deleted, and the only thing allowed to +# overwrite it is hive_refresh_legacy_fallback (#17862), which copies a +# newer, validated runtime config over it. On Docker/LXC it can be the +# only copy of the live config, so nothing else may touch it. +if grep -Eq 'rm[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT"; then + echo " FAIL: entrypoint removes the legacy runtime config" FAIL=$((FAIL + 1)) else - echo " PASS: legacy runtime config is never moved or removed" + echo " PASS: legacy runtime config is never removed" PASS=$((PASS + 1)) fi +REFRESH="$(sed -n '/^hive_refresh_legacy_fallback() {/,/^}/p' "$ENTRYPOINT")" +mv_total="$(grep -cE 'mv[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT" || true)" +mv_in_refresh="$(grep -cE 'mv[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' <<<"$REFRESH" || true)" +check "legacy runtime config is only overwritten inside hive_refresh_legacy_fallback" "$mv_total" "$mv_in_refresh" + +# ── #17862: the legacy fallback is refreshed on clean boots and WARNs loudly ── +# +# The incident: a kill mid-save left a 0-byte hive.yaml.runtime, the resolver +# fell back to a hive.yaml.bak frozen three months earlier, and the hive +# booted a stale ACMM level with no indication in the log. Two properties +# close it: (a) a clean boot from the new name refreshes the legacy copy so it +# is at most one boot old, and (b) a fallback is announced on stderr. + +echo +echo "=== #17862: legacy fallback freshness and WARN ===" + +# resolve_stderr <new-content> <legacy-content> — stderr the resolver emits. +resolve_stderr() { + local new_content="$1" legacy_content="$2" + local dir + dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$dir'" RETURN + HIVE_CONFIG_RUNTIME="$dir/hive.yaml.runtime" + HIVE_CONFIG_RUNTIME_LEGACY="$dir/hive.yaml.bak" + [ -n "$new_content" ] && printf '%s' "$new_content" > "$HIVE_CONFIG_RUNTIME" + [ -n "$legacy_content" ] && printf '%s' "$legacy_content" > "$HIVE_CONFIG_RUNTIME_LEGACY" + [ "$new_content" = "EMPTY" ] && : > "$HIVE_CONFIG_RUNTIME" + export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY + (eval "$RESOLVER"; hive_runtime_config_read 2>&1 >/dev/null) || true +} + +# (b) Falling back is loud; the steady state is silent. +warn_out="$(resolve_stderr "EMPTY" "old: cfg")" +if grep -q 'WARN.*FALLING BACK to legacy' <<<"$warn_out" && grep -q '0 bytes' <<<"$warn_out"; then + echo " PASS: empty runtime + legacy fallback logs a loud WARN naming the truncation" + PASS=$((PASS + 1)) +else + echo " FAIL: fallback to legacy did not WARN (got: '$warn_out')" + FAIL=$((FAIL + 1)) +fi +check "steady-state boot from the new name is silent" "" "$(resolve_stderr "new: cfg" "old: cfg")" + +# (a) hive_refresh_legacy_fallback: the legacy copy is overwritten from a +# valid runtime file, never from an invalid one, and only when the new +# name is the boot source. +HARDEN_FN="$(sed -n '/^hive_harden_runtime_config() {/,/^}/p' "$ENTRYPOINT")" +refresh_case() { + # refresh_case <boot-source: new|legacy> <runtime-content> <legacy-content>; prints legacy content after. + local src="$1" rt="$2" lg="$3" dir + dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$dir'" RETURN + HIVE_CONFIG_RUNTIME="$dir/hive.yaml.runtime" + HIVE_CONFIG_RUNTIME_LEGACY="$dir/hive.yaml.bak" + HIVE_RUNTIME_USER="$(id -un)"; HIVE_RUNTIME_GROUP="$(id -gn)" + export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY HIVE_RUNTIME_USER HIVE_RUNTIME_GROUP + printf '%s' "$rt" > "$HIVE_CONFIG_RUNTIME" + printf '%s' "$lg" > "$HIVE_CONFIG_RUNTIME_LEGACY" + local arg="$HIVE_CONFIG_RUNTIME" + [ "$src" = legacy ] && arg="$HIVE_CONFIG_RUNTIME_LEGACY" + (eval "$HARDEN_FN"; eval "$REFRESH"; hive_refresh_legacy_fallback "$arg" >/dev/null 2>&1) || true + cat "$HIVE_CONFIG_RUNTIME_LEGACY" +} +# Contents are single-line here so $(...) newline stripping cannot bite. +GOOD_RT='{project: {org: acme}, acmm_level: 6}' +STALE_LG='{acmm_level: 3}' +check "clean boot from new name refreshes the legacy fallback" \ + "$GOOD_RT" "$(refresh_case new "$GOOD_RT" "$STALE_LG")" +check "boot from the legacy name leaves it untouched" \ + "$STALE_LG" "$(refresh_case legacy "$GOOD_RT" "$STALE_LG")" +if command -v python3 >/dev/null 2>&1 && python3 -c 'import yaml' 2>/dev/null; then + check "unparsable runtime config never overwrites the fallback" \ + "$STALE_LG" "$(refresh_case new 'project: [unterminated' "$STALE_LG")" + check "runtime config without a project section never overwrites the fallback" \ + "$STALE_LG" "$(refresh_case new '{acmm_level: 0}' "$STALE_LG")" +else + hive_test_skip "python3+yaml unavailable — cannot exercise the parse guard in hive_refresh_legacy_fallback" +fi # 8. Every write of the config path to the PVC targets the NEW name. Counted, # not just grepped: there are two such writes (the K8s post-merge snapshot diff --git a/src/pkg/config/atomic_write.go b/src/pkg/config/atomic_write.go new file mode 100644 index 0000000000..bdd969964d --- /dev/null +++ b/src/pkg/config/atomic_write.go @@ -0,0 +1,116 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// writeFileAtomic replaces the contents of path without ever exposing a +// truncated or partial file to a concurrent reader or to the next boot. +// +// Motivation (#17862): Save used O_WRONLY|O_TRUNC in place on the PVC runtime +// config. A SIGKILL (liveness probe) landing between the truncate and the +// write left a 0-byte /data/hive.yaml.runtime; the entrypoint's `-s` check +// then fell back to a months-stale /data/hive.yaml.bak and a hive booted +// with a three-month-old ACMM level and roster. +// +// Strategy: write data to a temp file in the TARGET's directory, fsync it, +// then rename it onto the target. Two constraints shape the details: +// +// - path may be a symlink (hosted spokes link /etc/hive/hive.yaml to the +// PVC runtime config, #17468). The rename must land on the link's +// TARGET, not replace the link with a regular file, so the final path is +// resolved with EvalSymlinks first. Pinned by TestSaveThroughSymlinkKeepsLink. +// - path may be a single-file Docker bind mount, where rename over the +// mount point fails (EBUSY) and would also swap the inode the host sees. +// When the rename fails, fall back to the previous in-place write so +// those deployments keep working exactly as before; they never had +// atomicity and lose nothing here. +// +// mode is applied to the temp file before the rename, so a pre-existing +// world-readable inode does not carry its bits through (WriteFile's mode +// only applies on create). The fallback path chmods explicitly for the same +// reason. +func writeFileAtomic(path string, data []byte, mode os.FileMode) error { + target := path + if resolved, err := filepath.EvalSymlinks(path); err == nil { + target = resolved + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resolving %s: %w", path, err) + } + + dir := filepath.Dir(target) + tmp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*") + if err != nil { + return fmt.Errorf("creating temp file in %s: %w", dir, err) + } + tmpPath := tmp.Name() + cleanup := func() { _ = os.Remove(tmpPath) } + + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("chmod temp file %s: %w", tmpPath, err) + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("writing temp file %s: %w", tmpPath, err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("syncing temp file %s: %w", tmpPath, err) + } + if err := tmp.Close(); err != nil { + cleanup() + return fmt.Errorf("closing temp file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, target); err != nil { + cleanup() + // Bind-mounted single file (or any filesystem that refuses the + // rename): keep the inode and write in place, as Save always did. + if inPlaceErr := writeFileInPlace(target, data, mode); inPlaceErr != nil { + return fmt.Errorf("renaming %s onto %s: %w (in-place fallback also failed: %v)", tmpPath, target, err, inPlaceErr) + } + return nil + } + syncDir(dir) + return nil +} + +// writeFileInPlace is the pre-#17862 O_TRUNC write, kept only as the fallback +// for targets that cannot be renamed over. It is NOT crash-safe. +func writeFileInPlace(target string, data []byte, mode os.FileMode) error { + f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Chmod(target, mode) +} + +// syncDir fsyncs a directory so the rename that just happened in it is +// durable. Best-effort: some filesystems (and all of Windows) refuse to open +// a directory for sync, and a failure here cannot un-do the rename. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} diff --git a/src/pkg/config/atomic_write_test.go b/src/pkg/config/atomic_write_test.go new file mode 100644 index 0000000000..dfdb561e23 --- /dev/null +++ b/src/pkg/config/atomic_write_test.go @@ -0,0 +1,288 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestWriteFileAtomicReplacesContentAndMode covers the plain case: an +// existing world-readable file is replaced wholesale and ends up with the +// requested mode, and no temp file is left behind. +func TestWriteFileAtomicReplacesContentAndMode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "hive.yaml.runtime") + if err := os.WriteFile(path, []byte("old: content\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFileAtomic(path, []byte("new: content\n"), 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new: content\n" { + t.Fatalf("content = %q, want new content", got) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 0600 (pre-existing 0644 bits must not survive)", fi.Mode().Perm()) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp-") { + t.Fatalf("temp file %s left behind", e.Name()) + } + } +} + +// TestWriteFileAtomicCreatesMissingFile pins that a missing target is +// created (Save's old create fallback) rather than failing on EvalSymlinks. +func TestWriteFileAtomicCreatesMissingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "hive.yaml") + if err := writeFileAtomic(path, []byte("a: 1\n"), 0o644); err != nil { + t.Fatalf("writeFileAtomic on missing file: %v", err) + } + if got, _ := os.ReadFile(path); string(got) != "a: 1\n" { + t.Fatalf("content = %q", got) + } +} + +// TestWriteFileAtomicThroughSymlinkKeepsLink is the #17468 constraint at the +// writer level: the rename must land on the link TARGET so the symlink the +// copy-config init container created survives, and the target carries the +// new bytes. +func TestWriteFileAtomicThroughSymlinkKeepsLink(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "data", "hive.yaml.runtime") + link := filepath.Join(dir, "etc", "hive.yaml") + for _, d := range []string{filepath.Dir(target), filepath.Dir(link)} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(target, []byte("old\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if err := writeFileAtomic(link, []byte("new\n"), 0o600); err != nil { + t.Fatalf("writeFileAtomic through link: %v", err) + } + fi, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Fatalf("%s is no longer a symlink — rename replaced the link instead of its target", link) + } + if got, _ := os.ReadFile(target); string(got) != "new\n" { + t.Fatalf("target content = %q, want new", got) + } + // Nothing may be left in the link's directory: the temp file must be + // created next to the TARGET, or the rename would cross directories. + entries, _ := os.ReadDir(filepath.Dir(link)) + if len(entries) != 1 { + t.Fatalf("link dir has %d entries, want just the link", len(entries)) + } +} + +// TestSaveSurvivesKillMidWrite is the #17862 regression. It cannot SIGKILL +// the test process, so it pins the property that makes a kill harmless: +// at no point during Save does the runtime config path refer to a file +// with fewer bytes than the previous complete config. A temp-file+rename +// writer satisfies this trivially; the old O_TRUNC in-place write fails it +// because the file is observably 0 bytes between truncate and write. +// +// The check is made by racing a reader against repeated Saves and +// asserting it never observes an empty or partial file. +func TestSaveSurvivesKillMidWrite(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "hive.yaml") + runtimePath := filepath.Join(dir, "hive.yaml.runtime") + origOverlay, origRuntime := DashboardOverlayFile, RuntimeConfigFile + DashboardOverlayFile = filepath.Join(dir, "hive.yaml.dashboard") + RuntimeConfigFile = runtimePath + t.Cleanup(func() { DashboardOverlayFile, RuntimeConfigFile = origOverlay, origRuntime }) + + level := 6 + cfg := &Config{ + SourcePath: cfgPath, + Project: ProjectConfig{Org: "testorg", Name: "test"}, + AgentRuntimeConfig: AgentRuntimeConfig{ + Agents: map[string]AgentConfig{"scanner": {Role: "scanner"}}, + }, + ACMMLevel: &level, + } + if err := cfg.Save(); err != nil { + t.Fatalf("initial Save: %v", err) + } + full, err := os.ReadFile(runtimePath) + if err != nil || len(full) == 0 { + t.Fatalf("initial runtime config unreadable: %v", err) + } + + stop := make(chan struct{}) + bad := make(chan string, 1) + go func() { + for { + select { + case <-stop: + return + default: + } + for _, p := range []string{runtimePath, cfgPath} { + got, err := os.ReadFile(p) + if err != nil { + continue // a rename window may briefly ENOENT on some filesystems; never partial + } + if len(got) < len(full) { + select { + case bad <- p: + default: + } + return + } + } + } + }() + for i := 0; i < 200; i++ { + if err := cfg.Save(); err != nil { + t.Fatalf("Save %d: %v", i, err) + } + } + close(stop) + select { + case p := <-bad: + t.Fatalf("reader observed a truncated/partial config at %s during Save — write is not atomic", p) + default: + } +} + +func TestWriteFileAtomicMissingDirFails(t *testing.T) { + path := filepath.Join(t.TempDir(), "no-such-dir", "hive.yaml") + err := writeFileAtomic(path, []byte("x"), 0o600) + if err == nil { + t.Fatal("expected error when target directory does not exist") + } + if !strings.Contains(err.Error(), "creating temp file") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteFileAtomicResolveErrorIsReported(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "regular") + if err := os.WriteFile(file, []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + // Using a regular file as a path component makes EvalSymlinks fail with + // ENOTDIR, which is not ErrNotExist and must surface as a resolve error. + err := writeFileAtomic(filepath.Join(file, "hive.yaml"), []byte("x"), 0o600) + if err == nil { + t.Fatal("expected resolve error") + } + if !strings.Contains(err.Error(), "resolving") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteFileAtomicDanglingSymlinkIsReplaced(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(dir, "hive.yaml") + if err := os.Symlink(filepath.Join(dir, "missing-target"), link); err != nil { + t.Skipf("symlink not supported: %v", err) + } + if err := writeFileAtomic(link, []byte("fresh"), 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(link) + if err != nil { + t.Fatal(err) + } + if string(got) != "fresh" { + t.Fatalf("got %q", got) + } +} + +func TestWriteFileAtomicRenameFailureFallsBackAndReportsBoth(t *testing.T) { + dir := t.TempDir() + // A non-empty directory as the target: rename of a file onto it fails, + // and the in-place O_TRUNC fallback cannot open a directory for writing. + target := filepath.Join(dir, "hive.yaml") + if err := os.MkdirAll(filepath.Join(target, "child"), 0o700); err != nil { + t.Fatal(err) + } + err := writeFileAtomic(target, []byte("x"), 0o600) + if err == nil { + t.Fatal("expected rename + fallback failure") + } + if !strings.Contains(err.Error(), "in-place fallback also failed") { + t.Fatalf("unexpected error: %v", err) + } + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".hive.yaml.tmp-") { + t.Fatalf("temp file %s leaked after failure", e.Name()) + } + } +} + +func TestWriteFileInPlaceWritesAndSetsMode(t *testing.T) { + path := filepath.Join(t.TempDir(), "hive.yaml") + if err := os.WriteFile(path, []byte("old-longer-content"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFileInPlace(path, []byte("new"), 0o600); err != nil { + t.Fatalf("writeFileInPlace: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new" { + t.Fatalf("got %q, want truncated rewrite", got) + } + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 600", st.Mode().Perm()) + } +} + +func TestWriteFileInPlaceOpenFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "hive.yaml") + if err := writeFileInPlace(path, []byte("x"), 0o600); err == nil { + t.Fatal("expected open failure for missing directory") + } +} + +func TestSyncDirToleratesMissingDir(t *testing.T) { + syncDir(filepath.Join(t.TempDir(), "nope")) + syncDir(t.TempDir()) +} + +func TestSourceConfigModePreservesExistingOrDefaults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "hive.yaml") + if err := os.WriteFile(path, []byte("a: 1\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := sourceConfigMode(path); got != 0o600 { + t.Fatalf("existing mode = %o, want 600", got) + } + if got := sourceConfigMode(filepath.Join(dir, "absent.yaml")); got != 0o644 { + t.Fatalf("default mode = %o, want 644", got) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index bbeb01a2f8..6c80ab373b 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -7755,18 +7755,14 @@ func (c *Config) saveLocked() error { // layer this Save is about to persist. overlayErr := c.saveDashboardOverlay() - // Open the existing file (preserving its inode) rather than creating a - // temp file and renaming. Rename breaks Docker bind mounts because it - // replaces the inode — the host file is never updated, so acmm_level - // and other runtime changes are lost on container restart. + // Replace the primary config atomically (temp file + fsync + rename onto + // the resolved target) — see writeFileAtomic for why that is safe on the + // #17468 symlink and still works on single-file Docker bind mounts. // - // #17468: the same in-place write is what keeps hosted spokes durable. - // There the copy-config init container makes SourcePath - // (/etc/hive/hive.yaml, an emptyDir) a symlink to RuntimeConfigFile on - // the PVC; O_WRONLY|O_TRUNC follows the link and writes the PVC file, - // whereas a temp-file+rename would replace the link with a regular - // emptyDir file and silently revert the fix. Pinned by - // TestSaveThroughSymlinkKeepsLink — do not switch this to a rename. + // #17862: the previous O_WRONLY|O_TRUNC in-place write could be killed + // between truncate and write (liveness probe on a crash-looping hive), + // leaving a 0-byte runtime config that the entrypoint skipped in favour + // of a months-stale hive.yaml.bak. Pinned by TestSaveSurvivesKillMidWrite. // // #3961: a source-path failure must NOT abort the save. On deployments // that mount the config read-only (a ConfigMap mounted straight at @@ -7780,23 +7776,8 @@ func (c *Config) saveLocked() error { // writing the runtime layer, and report success iff the state will actually // survive a restart. var srcErr error - f, err := os.OpenFile(c.SourcePath, os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { - // File may not exist yet — fall back to create. Continue below so - // the PVC backup and dashboard overlay are still written. - if writeErr := os.WriteFile(c.SourcePath, data, 0o644); writeErr != nil { - srcErr = fmt.Errorf("writing config (create fallback): %w", writeErr) - } - } else { - if _, err := f.Write(data); err != nil { - _ = f.Close() // best-effort cleanup; the write error is what's recorded - srcErr = fmt.Errorf("writing config: %w", err) - } else if err := f.Sync(); err != nil { - _ = f.Close() // best-effort cleanup; the sync error is what's recorded - srcErr = fmt.Errorf("syncing config: %w", err) - } else if err := f.Close(); err != nil { - srcErr = fmt.Errorf("closing config: %w", err) - } + if err := writeFileAtomic(c.SourcePath, data, sourceConfigMode(c.SourcePath)); err != nil { + srcErr = fmt.Errorf("writing config: %w", err) } // Persist the runtime config to the PVC. In K8s this is the boot input @@ -7820,11 +7801,11 @@ func (c *Config) saveLocked() error { if guardLivePVCPathUnderTest(runtimePath, defaultRuntimeConfigFile, "RuntimeConfigFile") { // Not written, so not persisted: keep the primary-path error visible. runtimeErr = errLivePVCPathGuarded - } else if err := os.WriteFile(runtimePath, data, 0o600); err != nil { + } else if err := writeFileAtomic(runtimePath, data, 0o600); err != nil { // Common cause: init container created the file as root, runtime user // can't overwrite. Remove and retry so runtime state is not silently lost. - _ = os.Remove(runtimePath) // best-effort; the retry's own WriteFile error is what's recorded below - if retryErr := os.WriteFile(runtimePath, data, 0o600); retryErr != nil { + _ = os.Remove(runtimePath) // best-effort; the retry's own error is what's recorded below + if retryErr := writeFileAtomic(runtimePath, data, 0o600); retryErr != nil { runtimeErr = retryErr log.Printf("[config] warning: failed to write PVC runtime config to %s (even after remove): %v", runtimePath, retryErr) } else { @@ -7832,12 +7813,6 @@ func (c *Config) saveLocked() error { } } else { log.Printf("[config] PVC runtime config written to %s", runtimePath) - // os.WriteFile's mode only applies when it CREATES the file; a - // pre-existing world-readable inode (every hive deployed before - // this fix) keeps its old 0644 bits, so tighten explicitly. - if chmodErr := os.Chmod(runtimePath, 0o600); chmodErr != nil { - log.Printf("[config] warning: failed to tighten permissions on %s: %v", runtimePath, chmodErr) - } } if srcErr == nil { @@ -7856,6 +7831,16 @@ func (c *Config) saveLocked() error { return srcErr } +// sourceConfigMode returns the mode Save should give the primary config +// path: the existing file's permission bits when it exists (an operator may +// have tightened them), else the historical 0644 default. +func sourceConfigMode(path string) os.FileMode { + if fi, err := os.Stat(path); err == nil { + return fi.Mode().Perm() + } + return 0o644 +} + // RuntimeConfigFile is where Save() persists the full runtime config on the // PVC. Its role differs by environment, which is exactly why the old // hive.yaml.bak name was misleading enough to cost debugging time: @@ -7880,11 +7865,12 @@ const defaultRuntimeConfigFile = "/data/hive.yaml.runtime" // RuntimeConfigFileLegacy is the pre-rename name of RuntimeConfigFile. // -// It is READ as a fallback and never written, renamed or removed: ~51 live -// hives carry only this file, and on Docker/LXC it is the single copy of -// their live configuration. Mutating it at boot could lose owner -// customisations with no warning, so the migration is copy-forward only — -// readers prefer RuntimeConfigFile and fall back to this one. +// It is READ as a fallback and never written by Save: ~51 live hives carry +// only this file, and on Docker/LXC it is the single copy of their live +// configuration. Readers prefer RuntimeConfigFile and fall back to this one. +// The entrypoint refreshes it from a good RuntimeConfigFile on every clean +// boot (#17862) — a fallback frozen at the Aug 2026 rename booted a +// months-old level/roster when a kill mid-save truncated the runtime file. // // Removable one release after every live hive has written the new name. const RuntimeConfigFileLegacy = "/data/hive.yaml.bak" diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index 8ef3b3db00..b5de4a99ca 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -3464,11 +3464,12 @@ spec: # Steady state (#17468): /etc/hive is an emptyDir, so a regular file # there is a throwaway cache and operator edits to it vanished on # restart. Symlink it to the PVC runtime config instead, so the path - # every runbook names IS the persisted boot source. Config.Save writes - # the config path in place (O_TRUNC, no rename), so the link survives - # saves. Only the NEW name is linked: the legacy .bak is never written - # (config.RuntimeConfigFileLegacy), and linking it would make every - # save write it. + # every runbook names IS the persisted boot source. Config.Save + # resolves the link and renames its temp file onto the TARGET + # (#17862), so the link survives saves. Only the NEW name is linked: + # the legacy .bak is refreshed by the entrypoint from a good runtime + # file on clean boots (never written by Save), and linking it would + # make every save write it. command: ["sh", "-c", "if [ -s /data/hive.yaml.runtime ]; then echo runtime-config-exists-for-recovery; ln -sf /data/hive.yaml.runtime /etc/hive/hive.yaml && echo config-path-linked-to-runtime || echo config-path-link-failed; elif [ -s /data/hive.yaml.bak ]; then echo legacy-runtime-config-exists-for-recovery; else cp /etc/hive-seed/hive.yaml /etc/hive/hive.yaml && echo configmap-copied-first-boot; fi"] volumeMounts: - name: config From 3ff40077a0ef92fb366816cae2f34e8e0c2b8fd3 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 08:51:41 +0000 Subject: [PATCH 075/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.231.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.231.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-voice-sources.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-voice-sources.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9b669559..60d41bcc78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.231.0) + +### Added + +- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. + ## 2026-11-12 (v5.230.0) ### Added diff --git a/changelog.d/added-inbox-voice-sources.md b/changelog.d/added-inbox-voice-sources.md deleted file mode 100644 index a375f8cb62..0000000000 --- a/changelog.d/added-inbox-voice-sources.md +++ /dev/null @@ -1 +0,0 @@ -- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. From 1782cec7c74621339760feb182b4aa9d7f9885be Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:13:00 +0100 Subject: [PATCH 076/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20wait=20for=20mark?= =?UTF-8?q?etplace=20refresh=20completion=20in=20auth=20test=20(#17868)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wait for a new refresh result rather than interpreting an idle refresher as proof that the newly accepted asynchronous run has finished. Hive-Run: hivecommons/hive#17865 Hive-Plan: marketplace-refresh-flake Hive-Spec: marketplace-refresh-flake#async-completion Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> --- src/pkg/hub/marketplace_test.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index e28ba73458..b364fea6dd 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -870,18 +870,29 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { // Spoke full run: 202 once per hour across callers; admin unaffected. rf := s.refresher() - waitIdle := func() { + waitRefresh := func(previous *marketplaceRefreshResult) { + t.Helper() testutil.Eventually(t, 5*time.Second, func() bool { rf.mu.Lock() defer rf.mu.Unlock() - return !rf.running + return rf.last != previous && !rf.running }, "background refresh did not finish") } - rec = mktDo(s, spoke("/api/marketplace/refresh")) - if rec.Code != 202 { - t.Fatalf("spoke full: %d %s", rec.Code, rec.Body.String()) + refreshAndWait := func(req *http.Request, description string) { + t.Helper() + rf.mu.Lock() + previous := rf.last + rf.mu.Unlock() + rec := mktDo(s, req) + if rec.Code != http.StatusAccepted { + t.Fatalf("%s: %d %s", description, rec.Code, rec.Body.String()) + } + // Even a later fatal assertion must not restore fixture globals while + // this accepted refresh is still starting or running. + t.Cleanup(func() { waitRefresh(previous) }) + waitRefresh(previous) } - waitIdle() + refreshAndWait(spoke("/api/marketplace/refresh"), "spoke full") rf.mu.Lock() trigger := rf.last.Trigger rf.mu.Unlock() @@ -892,15 +903,9 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { if rec.Code != 429 || !strings.Contains(rec.Body.String(), "try again in") { t.Fatalf("spoke full again: %d %s", rec.Code, rec.Body.String()) } - if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh")); rec.Code != 202 { - t.Fatalf("admin full while spoke limited: %d %s", rec.Code, rec.Body.String()) - } - waitIdle() + refreshAndWait(admin(http.MethodPost, "/api/marketplace/refresh"), "admin full while spoke limited") clock = clock.Add(marketplaceFullRefreshInterval) - if rec = mktDo(s, spoke("/api/marketplace/refresh")); rec.Code != 202 { - t.Fatalf("spoke full after interval: %d", rec.Code) - } - waitIdle() + refreshAndWait(spoke("/api/marketplace/refresh"), "spoke full after interval") // Verbose status: anon 401, spoke and admin get the failing list. if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil)); rec.Code != 401 { From e625aee85a64c226b4c048204948f92e28747f40 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 09:22:46 +0000 Subject: [PATCH 077/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.231.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.231.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ changelog.d/changed-17863-pr-comment-quartet.md | 1 - changelog.d/fixed-17862-atomic-runtime-config-save.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/changed-17863-pr-comment-quartet.md delete mode 100644 changelog.d/fixed-17862-atomic-runtime-config-save.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d41bcc78..5c956ab5fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.231.1) + +### Changed + +- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `<!-- hive-pr-overlap -->` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `<!-- hive-mutation -->` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. + +### Fixed + +- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). + ## 2026-11-12 (v5.231.0) ### Added diff --git a/changelog.d/changed-17863-pr-comment-quartet.md b/changelog.d/changed-17863-pr-comment-quartet.md deleted file mode 100644 index eb5412ae90..0000000000 --- a/changelog.d/changed-17863-pr-comment-quartet.md +++ /dev/null @@ -1 +0,0 @@ -- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `<!-- hive-pr-overlap -->` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `<!-- hive-mutation -->` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. diff --git a/changelog.d/fixed-17862-atomic-runtime-config-save.md b/changelog.d/fixed-17862-atomic-runtime-config-save.md deleted file mode 100644 index 0dad10650e..0000000000 --- a/changelog.d/fixed-17862-atomic-runtime-config-save.md +++ /dev/null @@ -1 +0,0 @@ -- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). From 95cfe06c6930a433ebd5845e8050615f8a4a8c5c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:55:44 +0000 Subject: [PATCH 078/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20Inbox=20voice=20?= =?UTF-8?q?sources=20page=20+=20quickstart=20link=20(#17899)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: add Inbox voice sources page and quickstart links Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 📖 docs: use .row utility for Inbox quickstart link group TestStyleRatchet counts inline style= attributes in static/index.html; the new link group pushed it over the 1403 baseline. The existing .row utility (flex, gap var(--sp-4)) renders the same. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: link #17882 from the changelog fragment Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-inbox-voice-sources-docs.md | 1 + src/pkg/dashboard/static-src/js/knowledge.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/added-inbox-voice-sources-docs.md diff --git a/changelog.d/added-inbox-voice-sources-docs.md b/changelog.d/added-inbox-voice-sources-docs.md new file mode 100644 index 0000000000..02a0ce3d1f --- /dev/null +++ b/changelog.d/added-inbox-voice-sources-docs.md @@ -0,0 +1 @@ +- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 10fdd87fd5..ac6380b949 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2454,7 +2454,7 @@ const connectors = (_kcState.connectors || []).filter(c => c.type === type); if (!_kbIntegrationEditName || !connectors.some(c => c.name === _kbIntegrationEditName)) _kbIntegrationEditName = connectors[0]?.name || ''; const edited = connectors.find(c => c.name === _kbIntegrationEditName) || { name: '', type, enabled: true, layer: 'project', interval: '', scope: {}, auth: {} }; - let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></div></div>'; + let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><span class="row">' + (type === 'plaud' ? '<a href="/docs/inbox-voice-sources.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Inbox quickstart ↗</a>' : '') + '<a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></span></div></div>'; html += '<div class="row-between mb-4"><div class="section-label m-0">Configured entries</div>' + (owner ? '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationNew" data-arg0="' + escapeHtml(type) + '">Add ' + escapeHtml(info.label) + '</button>' : '') + '</div>'; html += connectors.length ? connectors.map(c => kbConnectorRow(type, c, owner)).join('') : '<div class="empty-dashed mb-4">No ' + escapeHtml(info.label) + ' connector configured yet.</div>'; html += owner ? kbConnectorFormHTML(type, edited, !!edited.name) : '<p class="muted-note">Only the hive owner can edit connector configuration.</p>'; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 90af378a72..dc5e519981 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46000,7 +46000,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const connectors = (_kcState.connectors || []).filter(c => c.type === type); if (!_kbIntegrationEditName || !connectors.some(c => c.name === _kbIntegrationEditName)) _kbIntegrationEditName = connectors[0]?.name || ''; const edited = connectors.find(c => c.name === _kbIntegrationEditName) || { name: '', type, enabled: true, layer: 'project', interval: '', scope: {}, auth: {} }; - let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></div></div>'; + let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><span class="row">' + (type === 'plaud' ? '<a href="/docs/inbox-voice-sources.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Inbox quickstart ↗</a>' : '') + '<a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></span></div></div>'; html += '<div class="row-between mb-4"><div class="section-label m-0">Configured entries</div>' + (owner ? '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationNew" data-arg0="' + escapeHtml(type) + '">Add ' + escapeHtml(info.label) + '</button>' : '') + '</div>'; html += connectors.length ? connectors.map(c => kbConnectorRow(type, c, owner)).join('') : '<div class="empty-dashed mb-4">No ' + escapeHtml(info.label) + ' connector configured yet.</div>'; html += owner ? kbConnectorFormHTML(type, edited, !!edited.name) : '<p class="muted-note">Only the hive owner can edit connector configuration.</p>'; From 36943e91fb8e7b816962085a5fc3331ab8347326 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:08:24 +0000 Subject: [PATCH 079/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20Settings=20?= =?UTF-8?q?=E2=86=92=20Inbox=20section=20(default=20repo,=20dibs=20URL,=20?= =?UTF-8?q?rules)=20(#17901)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: Settings → Inbox section for default repo, dibs URL and rules Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> #17885 landed inbox.* config, validation, parity allowlist and /api/inbox/settings but no Settings tab section. Add Settings → Inbox reusing the Inbox drawer form and endpoint. Part of #17883. * 🐛 dashboard: scope the Inbox settings form to its active root The Inbox settings form renders twice — in the Inbox page modal and inline under Settings → Inbox — and both can be mounted at once: the settings tab stays in the DOM and the closed modal keeps its last body. inboxReadRules collected .inbox-rule rows from every #inbox-settings-rules, the repo/dibs inputs were read via getElementById (first in document) and the inline sniff kept the modal open on save, so saving from the modal after visiting Settings → Inbox posted the inline form's values and left the dialog up. Wrap the form in .inbox-settings-form[data-inbox-form=inline|modal], replace the duplicated ids with data-inbox-field attributes, and resolve one active root per operation (the modal while open, otherwise the inline section). Read, render, save and the close-modal decision all go through that root. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/added-inbox-settings-section.md | 1 + .../dashboard-sections/workflow-and-agents.md | 1 + src/docs/inbox-voice-sources.md | 3 +- .../dashboard/inbox_settings_section_test.go | 34 +++++++ .../dashboard/static-src/js/config-state.js | 3 +- .../dashboard/static-src/js/config-tabs.js | 2 + src/pkg/dashboard/static-src/js/inbox.js | 84 +++++++++++++---- src/pkg/dashboard/static/index.html | 89 ++++++++++++++----- 8 files changed, 176 insertions(+), 41 deletions(-) create mode 100644 changelog.d/added-inbox-settings-section.md create mode 100644 src/pkg/dashboard/inbox_settings_section_test.go diff --git a/changelog.d/added-inbox-settings-section.md b/changelog.d/added-inbox-settings-section.md new file mode 100644 index 0000000000..7199a8ab60 --- /dev/null +++ b/changelog.d/added-inbox-settings-section.md @@ -0,0 +1 @@ +- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index c49a26bbd1..bb9ec25b24 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -213,6 +213,7 @@ Voice recordings and dictations waiting for you to file them as an RFC, send to - The sidebar count and the summary count items with no destination yet and not marked "Knowledge only". - Each item shows one pill per destination it was sent to: queued, running, filed (with a link) or failed. - Auto-routing rules run once per new item after a sync; the first matching rule wins. +- The default RFC repository, dibs URL and rules can also be edited under Settings → Inbox. **What it is good for.** Record an idea on your recorder, then turn it into an RFC with child issues (→ RFC), a dibs idea (→ dibs), or just keep it searchable (Knowledge only) without leaving the dashboard. See [Inbox and voice sources](../inbox-voice-sources.md). diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md index bb5c95792f..4f689aa9cd 100644 --- a/src/docs/inbox-voice-sources.md +++ b/src/docs/inbox-voice-sources.md @@ -36,7 +36,8 @@ see the issue linked from the changelog entry. - **Auto-routing rules** (⚙ Auto-routing in the Inbox, or `inbox.rules`): `source?`, `title_prefix` or `tag`, `destination`, `repo?`. Rules run once for each item that arrives after a connector sync; the first match wins; - items already in the Inbox are never re-routed. + items already in the Inbox are never re-routed. The same editor, with the + default RFC repository and dibs URL, is under Settings → Inbox. ## Configuration diff --git a/src/pkg/dashboard/inbox_settings_section_test.go b/src/pkg/dashboard/inbox_settings_section_test.go new file mode 100644 index 0000000000..147803693d --- /dev/null +++ b/src/pkg/dashboard/inbox_settings_section_test.go @@ -0,0 +1,34 @@ +package dashboard + +import ( + "strings" + "testing" +) + +func TestSettingsInboxSectionWiring(t *testing.T) { + raw, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatal(err) + } + page := string(raw) + for _, want := range []string{ + "'Knowledge', 'Inbox', 'Project Observability'", + "case 'Inbox': return renderGovInbox();", + "function renderGovInbox", + "function loadInboxSettingsInline", + "loadInboxSettingsInline();", + "inbox-settings-inline", + "yamlHint('inbox.spek.repo')", + "yamlHint('inbox.dibs.url')", + "yamlHint('inbox.rules')", + "Default RFC repository", + "data-inbox-field=\"dibs-url\"", + "function inboxSettingsRoot", + "data-inbox-form=", + "key: 'inbox.spek.repo'", + } { + if !strings.Contains(page, want) { + t.Errorf("Settings → Inbox wiring missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 162cf7e258..c3b76f8bfa 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -23,7 +23,7 @@ // which is read-only hub-synced user/role data. It sits next to Model // Gateways because both are backend configuration. const GOVERNOR_BOB_TAB = 'Bob'; - const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; + const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Inbox', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; const PIPELINE_STAGES = ['resolve-beads', 'track-prs', 'stale-check', 'repo-scan', 'coverage-gate', 'prompt-compose', 'budget-check', 'api-collect', 'final-compose']; const PIPELINE_STAGE_TIPS = { 'resolve-beads': 'Fetch and inject knowledge beads (facts, patterns, gotchas) relevant to the agent\'s current task context.', @@ -82,6 +82,7 @@ { tab: 'Labels', section: 'Labels', label: 'Adoption of a maintainer\'s PR', key: 'governor.labels.stage.adoption.maintainer_policy', text: 'adoption maintainer policy adopt ask never maintainer own pr clanker disconnected offline adopt action governor.labels.stage.adoption.maintainer_policy' }, { tab: 'Labels', section: 'Labels', label: 'Contributor nudge', key: 'governor.labels.stage.contributor_nudge.enabled', text: 'contributor nudge returned-to-contributor red ci fix-pr relay interrupt renudge presence window governor.labels.stage.contributor_nudge.enabled governor.labels.stage.contributor_nudge.renudge_interval auto_merge.nudge_repeat_after governor.labels.stage.contributor_nudge.ci_failure_cooldown governor.labels.stage.presence_window' }, { tab: 'Labels', section: 'Labels', label: 'Adopt red contributor PRs', key: 'governor.labels.stage.adoption.enabled', text: 'adoption adopted-by-hive stage 074 red contributor pr grace online offline supersede takeover in place after offline unregistered heartbeat governor.labels.stage.adoption.enabled governor.labels.stage.contributor_grace_online governor.labels.stage.contributor_grace_offline governor.labels.stage.adoption.after_offline governor.labels.stage.adoption.after_unregistered governor.labels.stage.adoption.in_place governor.labels.stage.adoption.strict_attribution strict attribution governor.labels.stage.adoption.auto_merge auto merge adopted head hive authority governor.labels.stage.adoption.supersede.close_first close first superseded reopen copy failed governor.labels.stage.adoption.close_superseded close superseded on adopt on merge clanker authored opt-in label governor.labels.stage.adoption.offline_grace offline grace rebase failed twice fixer resolving conflicts governor.labels.stage.adoption.rebase_attempt_timeout rebase attempt timeout' }, + { tab: 'Inbox', section: 'Inbox', label: 'Inbox routing', key: 'inbox.spek.repo', lazy: true, text: 'inbox voice recordings plaud wispr default rfc repository spek dibs url auto-routing rules inbox.spek.repo inbox.dibs.url inbox.rules' }, { tab: 'Hub', section: 'Contributor Relay', label: 'Fix budget', key: 'contribute.max_parallel_fixes', text: 'fix budget parallel fixes focus window red pr fix-pr preempt spawn contribute.max_parallel_fixes contribute.min_focus_window' }, { tab: 'Logging', section: 'Logging', label: 'Governor logging', key: 'governor.logging.max_size_mb', text: 'governor logging directory level compression max size age backups governor.logging.compress governor.logging.dir governor.logging.level governor.logging.max_age_days governor.logging.max_backups governor.logging.max_size_mb' }, { tab: 'Advisory', section: 'Advisory', label: 'Advisory digest', key: 'governor.advisory.max_findings', text: 'advisory digest max findings show all staleness pr autoclose target linear issue retry backoff update interval rewrite floor changed digest governor.advisory.update_interval_s governor.advisory.digest_min_interval' }, diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index 03d69b03f3..b8ca800a08 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -53,6 +53,7 @@ // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); + if (tabId === 'Inbox') loadInboxSettingsInline(); // The same tab carries the upstream-watch editor and divergence view // (#9969, #11211); it loads from its own endpoint. if (tabId === 'Features') loadUpstreamWatch(); @@ -265,6 +266,7 @@ case 'Access': return renderGovSecurity(d.security || {}); case 'Features': return renderGovFeatures(d.features || {}, d.auto_merge || {}, d.review || {}); case 'Knowledge': return renderGovKnowledgeSharing(); + case 'Inbox': return renderGovInbox(); case 'Project Observability': return renderGovProjectObservability(d.project_observability || {}); case 'Advisory': return renderGovAdvisory(d.advisory || {}); case 'Work Source': return renderGovWorkSource(d.work_source || {}); diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 4f38662937..9d2233d788 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -210,33 +210,77 @@ '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; } - function inboxReadRules() { - return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + // The same form renders in the Inbox page modal and inline under + // Settings → Inbox, and both can sit in the DOM at once (the settings + // tab stays mounted; the closed modal keeps its last body). Every read + // and write therefore goes through the active form root: the modal + // while it is open, otherwise the inline section. + function inboxSettingsRoot() { + const overlay = document.getElementById('kb-modal-overlay'); + if (overlay && overlay.style.display !== 'none') { + const modal = overlay.querySelector('.inbox-settings-form'); + if (modal) return modal; + } + return document.querySelector('#inbox-settings-inline .inbox-settings-form'); + } + function inboxSettingsField(root, name) { + return root ? root.querySelector('[data-inbox-field="' + name + '"]') : null; + } + function inboxReadRules(root) { + root = root || inboxSettingsRoot(); + if (!root) return []; + return Array.from(root.querySelectorAll('.inbox-settings-rules .inbox-rule')).map(row => { const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; }); } - function inboxRenderRules() { - const host = document.getElementById('inbox-settings-rules'); + function inboxRenderRules(root) { + root = root || inboxSettingsRoot(); + const host = root ? root.querySelector('.inbox-settings-rules') : null; if (!host) return; host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; } - function inboxOpenSettings() { - const settings = (_inboxCache || {}).settings || {}; - _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); - const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + function inboxSettingsFormHtml(settings, inline) { + const buttons = inline + ? '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>' + : '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + return '<div class="inbox-settings-form" data-inbox-form="' + (inline ? 'inline' : 'modal') + '"><p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + '<div class="inbox-settings-grid">' + - '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + - '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" data-inbox-field="spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" data-inbox-field="dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + - '<div id="inbox-settings-rules"></div>' + - '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + - '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + - '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; - kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + '<div class="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" data-inbox-field="error"></div>' + buttons + '</div>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + kbShowModal('⚙ Inbox auto-routing', inboxSettingsFormHtml(settings, false), { width: '720px' }); inboxRenderRules(); } + // Settings → Inbox: the same form inline, loaded from /api/inbox/settings. + function renderGovInbox() { + return '<div class="config-section" data-config-section="Inbox"><h3>Inbox</h3>' + + '<p class="text-muted fs-base">Default RFC repository, dibs URL and auto-routing rules for voice recordings. ' + yamlHint('inbox.spek.repo') + ' ' + yamlHint('inbox.dibs.url') + ' ' + yamlHint('inbox.rules') + '</p>' + + '<div id="inbox-settings-inline"><p class="muted-xs">Loading…</p></div></div>'; + } + async function loadInboxSettingsInline() { + const host = document.getElementById('inbox-settings-inline'); + if (!host) return; + try { + const r = await fetch('/api/inbox/settings'); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _inboxCache = Object.assign({}, _inboxCache, { settings: data }); + _inboxDraftRules = (data.rules || []).map(x => Object.assign({}, x)); + host.innerHTML = inboxSettingsFormHtml(data, true); + inboxRenderRules(host.querySelector('.inbox-settings-form')); + } catch (e) { + host.innerHTML = '<p class="kc-error muted-xs">Inbox settings unavailable: ' + escapeHtml(e.message) + '</p>'; + } + } function inboxAddRule() { _inboxDraftRules = inboxReadRules(); _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); @@ -248,14 +292,16 @@ inboxRenderRules(); } async function inboxSaveSettings() { - const errEl = document.getElementById('inbox-settings-error'); - const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); - const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + const root = inboxSettingsRoot(); + if (!root) return; + const errEl = inboxSettingsField(root, 'error'); + const rules = inboxReadRules(root).map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (inboxSettingsField(root, 'spek-repo') || {}).value || '', dibs_url: (inboxSettingsField(root, 'dibs-url') || {}).value || '' }; try { const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); const data = await r.json().catch(() => ({})); if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); - kbCloseModal(); + if (root.dataset.inboxForm !== 'inline') kbCloseModal(); showToast('Inbox settings saved', 'success'); await fetchInbox(); } catch (e) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index dc5e519981..517974a26c 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46406,33 +46406,77 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; } - function inboxReadRules() { - return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + // The same form renders in the Inbox page modal and inline under + // Settings → Inbox, and both can sit in the DOM at once (the settings + // tab stays mounted; the closed modal keeps its last body). Every read + // and write therefore goes through the active form root: the modal + // while it is open, otherwise the inline section. + function inboxSettingsRoot() { + const overlay = document.getElementById('kb-modal-overlay'); + if (overlay && overlay.style.display !== 'none') { + const modal = overlay.querySelector('.inbox-settings-form'); + if (modal) return modal; + } + return document.querySelector('#inbox-settings-inline .inbox-settings-form'); + } + function inboxSettingsField(root, name) { + return root ? root.querySelector('[data-inbox-field="' + name + '"]') : null; + } + function inboxReadRules(root) { + root = root || inboxSettingsRoot(); + if (!root) return []; + return Array.from(root.querySelectorAll('.inbox-settings-rules .inbox-rule')).map(row => { const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; }); } - function inboxRenderRules() { - const host = document.getElementById('inbox-settings-rules'); + function inboxRenderRules(root) { + root = root || inboxSettingsRoot(); + const host = root ? root.querySelector('.inbox-settings-rules') : null; if (!host) return; host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; } - function inboxOpenSettings() { - const settings = (_inboxCache || {}).settings || {}; - _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); - const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + function inboxSettingsFormHtml(settings, inline) { + const buttons = inline + ? '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>' + : '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + return '<div class="inbox-settings-form" data-inbox-form="' + (inline ? 'inline' : 'modal') + '"><p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + '<div class="inbox-settings-grid">' + - '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + - '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" data-inbox-field="spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" data-inbox-field="dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + - '<div id="inbox-settings-rules"></div>' + - '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + - '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + - '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; - kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + '<div class="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" data-inbox-field="error"></div>' + buttons + '</div>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + kbShowModal('⚙ Inbox auto-routing', inboxSettingsFormHtml(settings, false), { width: '720px' }); inboxRenderRules(); } + // Settings → Inbox: the same form inline, loaded from /api/inbox/settings. + function renderGovInbox() { + return '<div class="config-section" data-config-section="Inbox"><h3>Inbox</h3>' + + '<p class="text-muted fs-base">Default RFC repository, dibs URL and auto-routing rules for voice recordings. ' + yamlHint('inbox.spek.repo') + ' ' + yamlHint('inbox.dibs.url') + ' ' + yamlHint('inbox.rules') + '</p>' + + '<div id="inbox-settings-inline"><p class="muted-xs">Loading…</p></div></div>'; + } + async function loadInboxSettingsInline() { + const host = document.getElementById('inbox-settings-inline'); + if (!host) return; + try { + const r = await fetch('/api/inbox/settings'); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _inboxCache = Object.assign({}, _inboxCache, { settings: data }); + _inboxDraftRules = (data.rules || []).map(x => Object.assign({}, x)); + host.innerHTML = inboxSettingsFormHtml(data, true); + inboxRenderRules(host.querySelector('.inbox-settings-form')); + } catch (e) { + host.innerHTML = '<p class="kc-error muted-xs">Inbox settings unavailable: ' + escapeHtml(e.message) + '</p>'; + } + } function inboxAddRule() { _inboxDraftRules = inboxReadRules(); _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); @@ -46444,14 +46488,16 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> inboxRenderRules(); } async function inboxSaveSettings() { - const errEl = document.getElementById('inbox-settings-error'); - const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); - const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + const root = inboxSettingsRoot(); + if (!root) return; + const errEl = inboxSettingsField(root, 'error'); + const rules = inboxReadRules(root).map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (inboxSettingsField(root, 'spek-repo') || {}).value || '', dibs_url: (inboxSettingsField(root, 'dibs-url') || {}).value || '' }; try { const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); const data = await r.json().catch(() => ({})); if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); - kbCloseModal(); + if (root.dataset.inboxForm !== 'inline') kbCloseModal(); showToast('Inbox settings saved', 'success'); await fetchInbox(); } catch (e) { @@ -55233,7 +55279,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> // which is read-only hub-synced user/role data. It sits next to Model // Gateways because both are backend configuration. const GOVERNOR_BOB_TAB = 'Bob'; - const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; + const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Inbox', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; const PIPELINE_STAGES = ['resolve-beads', 'track-prs', 'stale-check', 'repo-scan', 'coverage-gate', 'prompt-compose', 'budget-check', 'api-collect', 'final-compose']; const PIPELINE_STAGE_TIPS = { 'resolve-beads': 'Fetch and inject knowledge beads (facts, patterns, gotchas) relevant to the agent\'s current task context.', @@ -55292,6 +55338,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> { tab: 'Labels', section: 'Labels', label: 'Adoption of a maintainer\'s PR', key: 'governor.labels.stage.adoption.maintainer_policy', text: 'adoption maintainer policy adopt ask never maintainer own pr clanker disconnected offline adopt action governor.labels.stage.adoption.maintainer_policy' }, { tab: 'Labels', section: 'Labels', label: 'Contributor nudge', key: 'governor.labels.stage.contributor_nudge.enabled', text: 'contributor nudge returned-to-contributor red ci fix-pr relay interrupt renudge presence window governor.labels.stage.contributor_nudge.enabled governor.labels.stage.contributor_nudge.renudge_interval auto_merge.nudge_repeat_after governor.labels.stage.contributor_nudge.ci_failure_cooldown governor.labels.stage.presence_window' }, { tab: 'Labels', section: 'Labels', label: 'Adopt red contributor PRs', key: 'governor.labels.stage.adoption.enabled', text: 'adoption adopted-by-hive stage 074 red contributor pr grace online offline supersede takeover in place after offline unregistered heartbeat governor.labels.stage.adoption.enabled governor.labels.stage.contributor_grace_online governor.labels.stage.contributor_grace_offline governor.labels.stage.adoption.after_offline governor.labels.stage.adoption.after_unregistered governor.labels.stage.adoption.in_place governor.labels.stage.adoption.strict_attribution strict attribution governor.labels.stage.adoption.auto_merge auto merge adopted head hive authority governor.labels.stage.adoption.supersede.close_first close first superseded reopen copy failed governor.labels.stage.adoption.close_superseded close superseded on adopt on merge clanker authored opt-in label governor.labels.stage.adoption.offline_grace offline grace rebase failed twice fixer resolving conflicts governor.labels.stage.adoption.rebase_attempt_timeout rebase attempt timeout' }, + { tab: 'Inbox', section: 'Inbox', label: 'Inbox routing', key: 'inbox.spek.repo', lazy: true, text: 'inbox voice recordings plaud wispr default rfc repository spek dibs url auto-routing rules inbox.spek.repo inbox.dibs.url inbox.rules' }, { tab: 'Hub', section: 'Contributor Relay', label: 'Fix budget', key: 'contribute.max_parallel_fixes', text: 'fix budget parallel fixes focus window red pr fix-pr preempt spawn contribute.max_parallel_fixes contribute.min_focus_window' }, { tab: 'Logging', section: 'Logging', label: 'Governor logging', key: 'governor.logging.max_size_mb', text: 'governor logging directory level compression max size age backups governor.logging.compress governor.logging.dir governor.logging.level governor.logging.max_age_days governor.logging.max_backups governor.logging.max_size_mb' }, { tab: 'Advisory', section: 'Advisory', label: 'Advisory digest', key: 'governor.advisory.max_findings', text: 'advisory digest max findings show all staleness pr autoclose target linear issue retry backoff update interval rewrite floor changed digest governor.advisory.update_interval_s governor.advisory.digest_min_interval' }, @@ -56383,6 +56430,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); + if (tabId === 'Inbox') loadInboxSettingsInline(); // The same tab carries the upstream-watch editor and divergence view // (#9969, #11211); it loads from its own endpoint. if (tabId === 'Features') loadUpstreamWatch(); @@ -56595,6 +56643,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> case 'Access': return renderGovSecurity(d.security || {}); case 'Features': return renderGovFeatures(d.features || {}, d.auto_merge || {}, d.review || {}); case 'Knowledge': return renderGovKnowledgeSharing(); + case 'Inbox': return renderGovInbox(); case 'Project Observability': return renderGovProjectObservability(d.project_observability || {}); case 'Advisory': return renderGovAdvisory(d.advisory || {}); case 'Work Source': return renderGovWorkSource(d.work_source || {}); From 9c45c48df07cb32e146089ae05295c7aed66f705 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 10:24:22 +0000 Subject: [PATCH 080/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.232.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.232.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-voice-sources-docs.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-voice-sources-docs.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c956ab5fe..df3ae99e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.232.0) + +### Added + +- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). + ## 2026-11-12 (v5.231.1) ### Changed diff --git a/changelog.d/added-inbox-voice-sources-docs.md b/changelog.d/added-inbox-voice-sources-docs.md deleted file mode 100644 index 02a0ce3d1f..0000000000 --- a/changelog.d/added-inbox-voice-sources-docs.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). From 90b51a67e599b1b14e0371cd3fe5041162b5a9e0 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:30:07 +0000 Subject: [PATCH 081/223] =?UTF-8?q?=E2=9C=A8=20inbox:=20Plaud=20source=20o?= =?UTF-8?q?nto=20the=20Source=20interface=20(#17900)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ inbox: read Plaud recordings directly through the Source interface The Inbox only saw recordings through the facts the Plaud connector writes. Add a PlaudSource that lists/fetches straight from the connector (item id stays attr_plaud_file_id), preferred while Plaud is signed in with the fact-backed source as fallback. Refs #17877, part of #17875. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🐛 inbox: cache the direct Plaud listing off the poll path and keep fact fields PreferDirect.List spawned the Plaud MCP process on every Items/Find/Sweep call, so each dashboard poll paid for a process spawn plus list_files pagination and a slow spawn past the handler timeout silently flipped the UI back to facts. The direct listing is now cached for a TTL and refreshed by a single background goroutine with its own timeout; List serves the cache (or just facts while nothing is cached) and never blocks on Plaud. A failed refresh clears the cache and is not retried before the TTL. Direct items also dropped FactSlug/FactPath/Layer/Excerpt, hiding the Fact button and excerpt for recordings that do have facts. The fact listing is now merged by id onto the direct items, and facts Plaud no longer lists stay visible. Refs #17877. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 inbox: wait on the Plaud refresh with testutil.Eventually, not a sleep loop The sleep ratchet (TestSleepRatchet) rejects any new fixed time.Sleep in test files; poll the observable condition instead. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 inbox: note that direct-listed Plaud items may precede their fact Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../added-inbox-plaud-direct-source.md | 1 + src/cmd/hive/boot_inbox.go | 27 ++- src/docs/inbox-voice-sources.md | 6 +- src/pkg/inbox/plaudsource.go | 207 ++++++++++++++++++ src/pkg/inbox/plaudsource_test.go | 165 ++++++++++++++ src/pkg/knowledge/connector/plaud.go | 40 ++-- .../knowledge/connector/plaud_recordings.go | 83 +++++++ src/pkg/knowledge/connector/plaud_test.go | 27 +++ 8 files changed, 538 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-inbox-plaud-direct-source.md create mode 100644 src/pkg/inbox/plaudsource.go create mode 100644 src/pkg/inbox/plaudsource_test.go create mode 100644 src/pkg/knowledge/connector/plaud_recordings.go diff --git a/changelog.d/added-inbox-plaud-direct-source.md b/changelog.d/added-inbox-plaud-direct-source.md new file mode 100644 index 0000000000..161055f469 --- /dev/null +++ b/changelog.d/added-inbox-plaud-direct-source.md @@ -0,0 +1 @@ +- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index 552c650911..b30f16ec2b 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -28,7 +28,7 @@ func (b *boot) bootInbox(baseDir string) { var srcs []inbox.Source for _, typ := range sources { if configured[typ] { - srcs = append(srcs, &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ}) + srcs = append(srcs, b.inboxSource(baseDir, typ)) } } if len(srcs) == 0 { @@ -60,6 +60,31 @@ func (b *boot) bootInbox(baseDir string) { b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) } +// inboxSource returns the fact-backed source for typ. For Plaud it prefers +// reading recordings directly from the connector while it is signed in. +func (b *boot) inboxSource(baseDir, typ string) inbox.Source { + facts := &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ} + if typ != inbox.PlaudType || b.knowledgeConnectors == nil { + return facts + } + for _, c := range b.cfg.Knowledge.Connectors { + if c.Type != typ { + continue + } + conn, ok := b.knowledgeConnectors.Connector(c.Name) + reader, isReader := conn.(connector.PlaudReader) + if !ok || !isReader { + continue + } + direct := &inbox.PlaudSource{Name: c.Name, Reader: reader} + if auth, ok := conn.(connector.Authenticator); ok { + direct.Ready = func(ctx context.Context) bool { return auth.AuthStatus(ctx, false).Connected } + } + return &inbox.PreferDirect{Direct: direct, Facts: facts} + } + return facts +} + // inboxAfterSync runs the auto-routing rules once a voice connector synced. func (b *boot) inboxAfterSync(st connector.Status) { svc := b.inboxSvc diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md index 4f689aa9cd..4f71b60ba2 100644 --- a/src/docs/inbox-voice-sources.md +++ b/src/docs/inbox-voice-sources.md @@ -15,8 +15,10 @@ see the issue linked from the changelog entry. `wisprflow`). Connecting a source happens in **Knowledge → 🔌 Integrations**; that card shows only the connection state and an **Open Inbox** button. - **Item** — one recording, keyed `(source, item_id)` (for Plaud, the - `attr_plaud_file_id`). Every item **is already a Knowledge fact**; the Inbox - only tracks what you did with it. "📚 Fact" opens the fact in Knowledge. + `attr_plaud_file_id`). When the source is connected the Inbox lists + recordings directly from it, so an item may appear before the sync has + written its Knowledge fact; the Inbox only tracks what you did with it. + "📚 Fact" opens the fact in Knowledge once that fact exists. - **Destination** — where an item goes: - **→ RFC** (`spek_rfc`): the hive runs `spektacular voice --from stdin --title <title> --publish --repo <owner/repo>` on the transcript and diff --git a/src/pkg/inbox/plaudsource.go b/src/pkg/inbox/plaudsource.go new file mode 100644 index 0000000000..2443d1b293 --- /dev/null +++ b/src/pkg/inbox/plaudsource.go @@ -0,0 +1,207 @@ +package inbox + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// PlaudType is the Source type for Plaud recordings. +const PlaudType = connector.TypePlaud + +// PlaudSource reads recordings straight from the Plaud connector. Item.ID is +// the Plaud file id (the `attr_plaud_file_id` of the fact the same sync +// writes), so route state carries over between this source and FactSource. +type PlaudSource struct { + // Name is the knowledge.connectors[] entry name. + Name string + Reader connector.PlaudReader + // Ready reports whether the connector is signed in; nil means always. + Ready func(ctx context.Context) bool +} + +// Type implements Source. +func (p *PlaudSource) Type() string { return PlaudType } + +// Available reports whether the connector can be read right now. +func (p *PlaudSource) Available(ctx context.Context) bool { + return p.Reader != nil && (p.Ready == nil || p.Ready(ctx)) +} + +// List implements Source: newest first. +func (p *PlaudSource) List(ctx context.Context) ([]Item, error) { + recs, err := p.Reader.ListRecordings(ctx) + if err != nil { + return nil, err + } + out := make([]Item, 0, len(recs)) + for _, r := range recs { + out = append(out, Item{ + Source: PlaudType, + Connector: p.Name, + ID: r.ID, + Title: r.Title, + RecordedAt: r.RecordedAt, + DurationMS: r.DurationMS, + Tags: r.Tags, + }) + } + sortItems(out) + return out, nil +} + +// Fetch implements Source. +func (p *PlaudSource) Fetch(ctx context.Context, it Item) (Content, error) { + c, err := p.Reader.ReadRecording(ctx, it.ID) + if err != nil { + return Content{}, err + } + var md strings.Builder + md.WriteString("# " + it.Title + "\n") + if c.Summary != "" { + md.WriteString("\n## Summary\n\n" + c.Summary + "\n") + } + if c.Transcript != "" { + md.WriteString("\n## Transcript\n\n" + c.Transcript + "\n") + } + return Content{ + Title: it.Title, + Summary: c.Summary, + Transcript: c.Transcript, + Markdown: strings.TrimSpace(md.String()), + Provenance: PlaudType + "://recordings/" + it.ID, + }, nil +} + +// directListTTL bounds how long a direct Plaud listing is reused before it +// is refreshed in the background. Listing spawns the Plaud MCP process, so +// the dashboard poll must never pay for it. +const directListTTL = 5 * time.Minute + +// directListTimeout bounds one background direct listing. +const directListTimeout = 2 * time.Minute + +// PreferDirect lists through Direct while it is available and healthy and +// falls back to Facts otherwise, so the Inbox keeps working before Plaud is +// signed in or when the MCP server is unreachable. +// +// The direct listing is cached for TTL and refreshed in the background: +// List returns whatever is cached (or just facts while nothing is) and +// never blocks on the Plaud process. Direct items are merged with the facts +// of the same id so they keep FactSlug/FactPath/Layer/Excerpt. +type PreferDirect struct { + Direct *PlaudSource + Facts Source + // TTL overrides directListTTL; zero means the default. + TTL time.Duration + + mu sync.Mutex + direct []Item + fetched time.Time + refreshing bool +} + +// Type implements Source. +func (s *PreferDirect) Type() string { return s.Facts.Type() } + +// List implements Source. +func (s *PreferDirect) List(ctx context.Context) ([]Item, error) { + facts, err := s.Facts.List(ctx) + if !s.Direct.Available(ctx) { + return facts, err + } + direct, stale := s.cached() + if stale { + s.refreshAsync() + } + if direct == nil { + return facts, err + } + return mergeDirect(direct, facts), err +} + +// cached returns the cached direct listing (nil when none or when the last +// refresh failed) and whether it is due for a refresh. +func (s *PreferDirect) cached() (items []Item, stale bool) { + s.mu.Lock() + defer s.mu.Unlock() + ttl := s.TTL + if ttl <= 0 { + ttl = directListTTL + } + return s.direct, !s.refreshing && time.Since(s.fetched) >= ttl +} + +// refreshAsync starts one background Refresh unless one is already running. +func (s *PreferDirect) refreshAsync() { + s.mu.Lock() + if s.refreshing { + s.mu.Unlock() + return + } + s.refreshing = true + s.mu.Unlock() + go func() { + ctx, cancel := context.WithTimeout(context.Background(), directListTimeout) + defer cancel() + _ = s.Refresh(ctx) + }() +} + +// Refresh lists Direct now and caches the result. A failure clears the +// cache so List falls back to Facts, and is not retried before TTL. +func (s *PreferDirect) Refresh(ctx context.Context) error { + items, err := s.Direct.List(ctx) + s.mu.Lock() + defer s.mu.Unlock() + s.refreshing = false + s.fetched = time.Now() + if err != nil { + s.direct = nil + return err + } + if items == nil { + items = []Item{} + } + s.direct = items + return nil +} + +// mergeDirect overlays the fact fields onto direct items with the same id +// and appends facts Plaud no longer lists (outside its since_days window). +func mergeDirect(direct, facts []Item) []Item { + byID := make(map[string]Item, len(facts)) + for _, f := range facts { + byID[f.ID] = f + } + out := make([]Item, 0, len(direct)+len(facts)) + for _, it := range direct { + if f, ok := byID[it.ID]; ok { + it.FactSlug, it.FactPath, it.Layer, it.Excerpt = f.FactSlug, f.FactPath, f.Layer, f.Excerpt + if it.Title == "" { + it.Title = f.Title + } + delete(byID, it.ID) + } + out = append(out, it) + } + for _, f := range facts { + if _, ok := byID[f.ID]; ok { + out = append(out, f) + } + } + sortItems(out) + return out +} + +// Fetch implements Source: fact-backed items (FactPath set) read their fact, +// direct items read Plaud. +func (s *PreferDirect) Fetch(ctx context.Context, it Item) (Content, error) { + if it.FactPath != "" { + return s.Facts.Fetch(ctx, it) + } + return s.Direct.Fetch(ctx, it) +} diff --git a/src/pkg/inbox/plaudsource_test.go b/src/pkg/inbox/plaudsource_test.go new file mode 100644 index 0000000000..7fb9953e2a --- /dev/null +++ b/src/pkg/inbox/plaudsource_test.go @@ -0,0 +1,165 @@ +package inbox + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +type fakePlaud struct { + recs []connector.PlaudRecording + content connector.PlaudContent + listErr error + reads []string +} + +func (f *fakePlaud) ListRecordings(context.Context) ([]connector.PlaudRecording, error) { + return f.recs, f.listErr +} + +func (f *fakePlaud) ReadRecording(_ context.Context, id string) (connector.PlaudContent, error) { + f.reads = append(f.reads, id) + return f.content, nil +} + +func TestPlaudSourceListAndFetch(t *testing.T) { + older := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + newer := older.Add(24 * time.Hour) + fp := &fakePlaud{ + recs: []connector.PlaudRecording{ + {ID: "rec-1", Title: "Old", RecordedAt: older, DurationMS: 1000}, + {ID: "rec-2", Title: "New", RecordedAt: newer, Tags: []string{"voice"}}, + }, + content: connector.PlaudContent{Summary: "sum", Transcript: "words"}, + } + src := &PlaudSource{Name: "my-plaud", Reader: fp} + if src.Type() != "plaud" { + t.Fatalf("type = %q", src.Type()) + } + items, err := src.List(context.Background()) + if err != nil || len(items) != 2 { + t.Fatalf("items=%v err=%v", items, err) + } + if items[0].ID != "rec-2" || items[0].Connector != "my-plaud" || items[0].Source != "plaud" || items[0].Key() != "plaud/rec-2" { + t.Fatalf("first item = %+v", items[0]) + } + if items[1].DurationMS != 1000 || !items[0].RecordedAt.Equal(newer) { + t.Fatalf("mapping lost: %+v", items) + } + c, err := src.Fetch(context.Background(), items[0]) + if err != nil { + t.Fatal(err) + } + if c.Title != "New" || c.Summary != "sum" || c.Transcript != "words" || c.Provenance != "plaud://recordings/rec-2" { + t.Fatalf("content = %+v", c) + } + if len(fp.reads) != 1 || fp.reads[0] != "rec-2" { + t.Fatalf("reads = %v", fp.reads) + } +} + +func TestPreferDirectSelection(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + facts := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + fp := &fakePlaud{recs: []connector.PlaudRecording{{ID: "direct-1", Title: "Direct", RecordedAt: time.Date(2026, 11, 11, 8, 0, 0, 0, time.UTC)}}} + ready := false + direct := &PlaudSource{Name: "plaud", Reader: fp, Ready: func(context.Context) bool { return ready }} + src := &PreferDirect{Direct: direct, Facts: facts, TTL: time.Hour} + ctx := context.Background() + + items, err := src.List(ctx) + if err != nil || len(items) != 1 || items[0].ID != "rec-1" { + t.Fatalf("not signed in should use facts: %v %v", items, err) + } + if _, err := src.Fetch(ctx, items[0]); err != nil || len(fp.reads) != 0 { + t.Fatalf("fact item must not hit plaud: err=%v reads=%v", err, fp.reads) + } + + ready = true + if err := src.Refresh(ctx); err != nil { + t.Fatal(err) + } + items, err = src.List(ctx) + if err != nil || len(items) != 2 || items[0].ID != "direct-1" || items[1].ID != "rec-1" { + t.Fatalf("signed in should merge plaud with facts: %v %v", items, err) + } + if items[1].FactPath == "" || items[1].Excerpt == "" { + t.Fatalf("fact-only item lost fact fields: %+v", items[1]) + } + if _, err := src.Fetch(ctx, items[0]); err != nil || len(fp.reads) != 1 { + t.Fatalf("direct item should read plaud: err=%v reads=%v", err, fp.reads) + } + + fp.listErr = errors.New("mcp down") + if err := src.Refresh(ctx); err == nil { + t.Fatal("refresh should report the plaud error") + } + items, err = src.List(ctx) + if err != nil || len(items) != 1 || items[0].ID != "rec-1" { + t.Fatalf("plaud failure should fall back to facts: %v %v", items, err) + } +} + +// List must never pay for the Plaud process: the first signed-in call +// returns facts and kicks off one background refresh. +func TestPreferDirectListDoesNotBlock(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + facts := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + fp := &countingPlaud{recs: []connector.PlaudRecording{{ID: "rec-1", Title: "Direct title"}}, gate: make(chan struct{})} + src := &PreferDirect{Direct: &PlaudSource{Name: "plaud", Reader: fp}, Facts: facts, TTL: time.Hour} + ctx := context.Background() + + for i := 0; i < 2; i++ { + items, err := src.List(ctx) + if err != nil || len(items) != 1 || items[0].Title != "RFC: Voice inbox" { + t.Fatalf("list %d while plaud is slow should serve facts: %v %v", i, items, err) + } + } + fp.release() + items := testutil.EventuallyValue(t, 5*time.Second, func() ([]Item, bool) { + got, _ := src.List(ctx) + return got, len(got) == 1 && got[0].Title == "Direct title" + }, "background refresh never landed") + if n := fp.calls.Load(); n != 1 { + t.Fatalf("ListRecordings calls = %d, want 1 (single-flight, cached for TTL)", n) + } + it := items[0] + if it.FactSlug != "plaud-plaud-rec-1" || it.Layer != "personal" || it.FactPath == "" || it.Excerpt == "" { + t.Fatalf("direct item lost fact fields: %+v", it) + } +} + +type countingPlaud struct { + recs []connector.PlaudRecording + calls atomic.Int32 + gate chan struct{} + once sync.Once +} + +func (c *countingPlaud) release() { + c.once.Do(func() { + if c.gate != nil { + close(c.gate) + } + }) +} + +func (c *countingPlaud) ListRecordings(context.Context) ([]connector.PlaudRecording, error) { + c.calls.Add(1) + if c.gate != nil { + <-c.gate + } + return c.recs, nil +} + +func (c *countingPlaud) ReadRecording(context.Context, string) (connector.PlaudContent, error) { + return connector.PlaudContent{}, nil +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go index f0ea586090..02e959cbaf 100644 --- a/src/pkg/knowledge/connector/plaud.go +++ b/src/pkg/knowledge/connector/plaud.go @@ -238,21 +238,9 @@ func (c *plaudConnector) listFiles(ctx context.Context, client mcpClient, since // fetchPage builds the knowledge page for one recording. func (c *plaudConnector) fetchPage(ctx context.Context, client mcpClient, f plaudFile) (Page, error) { - noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": f.ID}) - transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": f.ID}) - if noteErr != nil && trErr != nil { - return Page{}, plaudAuthError(trErr) - } - if noteErr != nil { - noteText = "" - } - if trErr != nil { - transcriptText = "" - } - note := renderPlaudNote(noteText) - transcript := renderPlaudTranscript(transcriptText) - if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { - return Page{}, fmt.Errorf("no note or transcript available yet") + note, transcript, err := c.fetchNoteAndTranscript(ctx, client, f.ID) + if err != nil { + return Page{}, err } audioURL := "" if c.includeAudio() { @@ -427,3 +415,25 @@ func plaudTime(m map[string]any, keys ...string) time.Time { } return time.Time{} } + +// fetchNoteAndTranscript renders the note and transcript of one recording. +// It fails only when neither is available. +func (c *plaudConnector) fetchNoteAndTranscript(ctx context.Context, client mcpClient, id string) (note, transcript string, err error) { + noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": id}) + transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": id}) + if noteErr != nil && trErr != nil { + return "", "", plaudAuthError(trErr) + } + if noteErr != nil { + noteText = "" + } + if trErr != nil { + transcriptText = "" + } + note = renderPlaudNote(noteText) + transcript = renderPlaudTranscript(transcriptText) + if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { + return "", "", fmt.Errorf("no note or transcript available yet") + } + return note, transcript, nil +} diff --git a/src/pkg/knowledge/connector/plaud_recordings.go b/src/pkg/knowledge/connector/plaud_recordings.go new file mode 100644 index 0000000000..291acb2f32 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_recordings.go @@ -0,0 +1,83 @@ +package connector + +import ( + "context" + "sort" + "strings" + "time" +) + +// PlaudRecording is one recording as the Plaud MCP server lists it. ID is +// the Plaud file id, the same value written to the `plaud_file_id` fact attr. +type PlaudRecording struct { + ID string + Title string + RecordedAt time.Time + DurationMS int64 + Tags []string +} + +// PlaudContent is the rendered note and transcript of one recording. +type PlaudContent struct { + Summary string + Transcript string +} + +// PlaudReader reads recordings straight from Plaud without going through +// the facts a sync writes. The plaud connector implements it. +type PlaudReader interface { + // ListRecordings lists recent recordings, newest first. + ListRecordings(ctx context.Context) ([]PlaudRecording, error) + // ReadRecording returns the note and transcript of one recording. + ReadRecording(ctx context.Context, id string) (PlaudContent, error) +} + +var _ PlaudReader = (*plaudConnector)(nil) + +// ListRecordings lists recordings within the connector's since_days window, +// capped at max_files, newest first. +func (c *plaudConnector) ListRecordings(ctx context.Context) ([]PlaudRecording, error) { + client, err := c.dial(ctx) + if err != nil { + return nil, err + } + defer func() { _ = client.Close() }() + since := c.now().Add(-time.Duration(c.scopeInt("since_days", plaudDefaultSinceDays)) * 24 * time.Hour) + files, err := c.listFiles(ctx, client, since, c.scopeInt("max_files", plaudDefaultMaxFiles)) + if err != nil { + return nil, err + } + var tags []string + for _, t := range strings.Split(c.cfg.ScopeValue("tags"), ",") { + if t = strings.TrimSpace(t); t != "" { + tags = append(tags, t) + } + } + out := make([]PlaudRecording, 0, len(files)) + for _, f := range files { + rec := PlaudRecording{ID: f.ID, Title: strings.TrimSpace(f.Name), RecordedAt: f.StartAt.UTC(), DurationMS: f.DurationMS, Tags: tags} + if f.StartAt.IsZero() { + rec.RecordedAt = f.CreatedAt.UTC() + } + if rec.Title == "" { + rec.Title = "Plaud recording " + f.ID + } + out = append(out, rec) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].RecordedAt.After(out[j].RecordedAt) }) + return out, nil +} + +// ReadRecording fetches the note and transcript of one recording. +func (c *plaudConnector) ReadRecording(ctx context.Context, id string) (PlaudContent, error) { + client, err := c.dial(ctx) + if err != nil { + return PlaudContent{}, err + } + defer func() { _ = client.Close() }() + note, transcript, err := c.fetchNoteAndTranscript(ctx, client, id) + if err != nil { + return PlaudContent{}, err + } + return PlaudContent{Summary: strings.TrimSpace(note), Transcript: strings.TrimSpace(transcript)}, nil +} diff --git a/src/pkg/knowledge/connector/plaud_test.go b/src/pkg/knowledge/connector/plaud_test.go index eada7e56b8..c0f645393f 100644 --- a/src/pkg/knowledge/connector/plaud_test.go +++ b/src/pkg/knowledge/connector/plaud_test.go @@ -227,3 +227,30 @@ func TestPlaudRenderHelpers(t *testing.T) { t.Fatalf("auth error = %q", got) } } + +func TestPlaudListAndReadRecording(t *testing.T) { + pc := newTestPlaud(t, newPlaudTestServer(), map[string]string{"tags": "voice, ideas"}) + recs, err := pc.ListRecordings(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(recs) != 2 || recs[0].ID != "rec-2" || recs[1].ID != "rec-1" { + t.Fatalf("recs = %+v", recs) + } + if recs[1].Title != "Standup ideas" || recs[1].DurationMS != 125000 || len(recs[1].Tags) != 2 { + t.Fatalf("rec-1 = %+v", recs[1]) + } + if want := time.Date(2026, 11, 10, 8, 58, 0, 0, time.UTC); !recs[1].RecordedAt.Equal(want) { + t.Fatalf("recorded at = %v", recs[1].RecordedAt) + } + c, err := pc.ReadRecording(context.Background(), "rec-1") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(c.Summary, "ship plaud connector") || !strings.Contains(c.Transcript, "Hello hive") { + t.Fatalf("content = %+v", c) + } + if _, err := pc.ReadRecording(context.Background(), "rec-2"); err == nil { + t.Fatal("recording with no note or transcript should error") + } +} From 53a5ce03262e2ecd34b19d186236a1a03346b0b4 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 10:54:36 +0000 Subject: [PATCH 082/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.233.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.233.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-settings-section.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-settings-section.md diff --git a/CHANGELOG.md b/CHANGELOG.md index df3ae99e7e..f03c8e78de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.233.0) + +### Added + +- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. + ## 2026-11-12 (v5.232.0) ### Added diff --git a/changelog.d/added-inbox-settings-section.md b/changelog.d/added-inbox-settings-section.md deleted file mode 100644 index 7199a8ab60..0000000000 --- a/changelog.d/added-inbox-settings-section.md +++ /dev/null @@ -1 +0,0 @@ -- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. From 70018a1232f5812e0448d44f3ef18d30e861b6a2 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:14:06 +0000 Subject: [PATCH 083/223] =?UTF-8?q?=F0=9F=90=9B=20check-action-pins:=20onl?= =?UTF-8?q?y=20HTTP=20404/422=20is=20MISSING-SHA;=20retry=20transient=20AP?= =?UTF-8?q?I=20errors=20and=20report=20them=20as=20UNRESOLVED=20(#17905)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 check-action-pins: only 404/422 is MISSING-SHA; retry and report transient API errors as UNRESOLVED A rate limit, 5xx or dropped connection from gh api was printed as 'does not exist in that repository', which is a wrong verdict: the pin was fine and the visible next step (bump the pin) was the wrong one. Retry three times, then fail closed with a distinct UNRESOLVED line. The test double now answers a missing SHA with the real HTTP 422 text and gains a transient-error case that fails without this change. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 integration-governance: re-pin TRUSTED_SCRIPTS_TREE for the check-action-pins change src/scripts changed in this PR, so the enrolled .github closure must pin the new tree or the materializer contract test rejects it. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .github/scripts/integration-governance.cjs | 2 +- ...d-check-action-pins-transient-api-error.md | 1 + src/scripts/check-action-pins.sh | 39 +++++++++++++++++-- src/scripts/test-check-action-pins.sh | 35 +++++++++++++++-- 4 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-check-action-pins-transient-api-error.md diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 87ad4e2ce7..27e5a5c64b 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '1f1eb904839561af15c1772e6d9106a3b8d2b028'; +const TRUSTED_SCRIPTS_TREE = '08802cd3162cd8a179514afe3ddb5e088c90d5f0'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/changelog.d/fixed-check-action-pins-transient-api-error.md b/changelog.d/fixed-check-action-pins-transient-api-error.md new file mode 100644 index 0000000000..b158778fd9 --- /dev/null +++ b/changelog.d/fixed-check-action-pins-transient-api-error.md @@ -0,0 +1 @@ +- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED <repo>@<sha> … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). diff --git a/src/scripts/check-action-pins.sh b/src/scripts/check-action-pins.sh index e60dd905a0..a143d5f9e3 100755 --- a/src/scripts/check-action-pins.sh +++ b/src/scripts/check-action-pins.sh @@ -17,6 +17,7 @@ set -uo pipefail DIR="${1:-.github/workflows}" fail=0 +RESOLVE_ERR="" # Every `uses:` on a third-party action must be a 40-hex SHA, not a tag or # branch. A moving ref is a supply-chain hole; that part is shape-only and needs @@ -34,6 +35,29 @@ while IFS= read -r line; do # DIR, so `${line%%:*}` names the workflow and not its line number. done < <(grep -rHnE '^\s*(-\s*)?uses:\s*[a-zA-Z0-9._-]+/' "$DIR" 2>/dev/null | grep -v '^\s*#') +# resolve_pin OWNER/REPO SHA — 0 the commit exists, 1 the API says it does not +# (422 "No commit found for SHA" or 404 for the repository itself), 2 the API +# gave no verdict. Only a definite 404/422 may be reported as MISSING-SHA: +# a rate limit, a 5xx or a dropped connection says nothing about the pin, and +# calling it "does not exist" sent crux main red on a README-only commit whose +# setup-go pin was tagged v7.0.0 upstream all along. Transient errors are +# retried briefly, then reported as UNRESOLVED so a re-run — not a pin bump — +# is the visible next step. +resolve_pin() { + local repo="$1" sha="$2" err attempt + for attempt in 1 2 3; do + if err="$(gh api "repos/$repo/commits/$sha" --jq '.sha' 2>&1 >/dev/null)"; then + return 0 + fi + case "$err" in + *"(HTTP 404)"*|*"(HTTP 422)"*) return 1 ;; + esac + [ "$attempt" -lt 3 ] && sleep "$attempt" + done + RESOLVE_ERR="$(printf '%s' "$err" | head -1)" + return 2 +} + # Resolve each unique pin. Skipped without a token so the script stays usable # offline; CI always has one. if command -v gh >/dev/null 2>&1; then @@ -51,10 +75,17 @@ if command -v gh >/dev/null 2>&1; then [ -z "$ref" ] && continue sha="${ref#*@}"; path="${ref%@*}" repo="$(printf '%s' "$path" | cut -d/ -f1,2)" - if ! gh api "repos/$repo/commits/$sha" --jq '.sha' >/dev/null 2>&1; then - echo "MISSING-SHA $repo@$sha does not exist in that repository" - fail=1 - fi + resolve_pin "$repo" "$sha" + case "$?" in + 1) + echo "MISSING-SHA $repo@$sha does not exist in that repository" + fail=1 + ;; + 2) + echo "UNRESOLVED $repo@$sha: the GitHub API gave no verdict after 3 attempts ($RESOLVE_ERR); re-run, this is not a missing pin" + fail=1 + ;; + esac done <<<"$pins" fi diff --git a/src/scripts/test-check-action-pins.sh b/src/scripts/test-check-action-pins.sh index c796f1384b..0d70f4fcaa 100755 --- a/src/scripts/test-check-action-pins.sh +++ b/src/scripts/test-check-action-pins.sh @@ -48,14 +48,19 @@ assert_eq() { # Fake `gh`: only `gh api repos/<owner>/<repo>/commits/<sha> --jq .sha` is # ever called by the check. Every call is appended to $FAKE_GH_LOG; a call # whose SHA equals $FAKE_GH_MISSING fails like the real API does for a commit -# that does not exist in the repository. +# that does not exist in the repository (HTTP 422 on stderr), and one whose +# SHA equals $FAKE_GH_TRANSIENT fails like a rate limit (HTTP 403), which is +# not a verdict on the pin. BIN="$TMP/bin" mkdir -p "$BIN" cat >"$BIN/gh" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" case "$2" in - *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") + echo "gh: No commit found for SHA: ${2##*/} (HTTP 422)" >&2; exit 1 ;; + *"/commits/${FAKE_GH_TRANSIENT:-no-such-sha}") + echo "gh: API rate limit exceeded for installation ID 1 (HTTP 403)" >&2; exit 1 ;; esac printf '%s\n' "${2##*/}" exit 0 @@ -183,7 +188,8 @@ cat >"$BIN/gh" <<SH #!/usr/bin/env bash printf '%s\n' "\$*" >>"\${FAKE_GH_LOG:?}" case "\$2" in - */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) exit 1 ;; + */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) + echo "gh: No commit found for SHA: \${2##*/} (HTTP 422)" >&2; exit 1 ;; esac printf '%s\n' "\${2##*/}" exit 0 @@ -199,12 +205,33 @@ cat >"$BIN/gh" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" case "$2" in - *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") + echo "gh: No commit found for SHA: ${2##*/} (HTTP 422)" >&2; exit 1 ;; + *"/commits/${FAKE_GH_TRANSIENT:-no-such-sha}") + echo "gh: API rate limit exceeded for installation ID 1 (HTTP 403)" >&2; exit 1 ;; esac printf '%s\n' "${2##*/}" exit 0 SH +echo "case: an API error that is not 404/422 is not a missing SHA" +D="$TMP/transient" +workflow "$D" ci.yml <<EOF +jobs: + build: + steps: + - uses: actions/checkout@$SHA_A + - uses: actions/setup-go@$SHA_B # v7.0.0 +EOF +run_check transient FAKE_GH_TRANSIENT="$SHA_B" -- "$D" +assert_exit 1 "$RC" "an unresolved pin still fails closed" +assert_not_contains "$OUT" "MISSING-SHA" "a rate limit is not reported as a missing SHA" +assert_contains "$OUT" "UNRESOLVED actions/setup-go@$SHA_B" "reports the pin as UNRESOLVED instead" +assert_contains "$OUT" "(HTTP 403)" "quotes the API error" +assert_contains "$OUT" "action pin check FAILED" "prints the failure banner" +assert_eq 3 "$(grep -c "commits/$SHA_B" <<<"$GH_CALLS")" "the transient failure is retried three times" +assert_eq 1 "$(grep -c "commits/$SHA_A" <<<"$GH_CALLS")" "a pin that resolves first time is not retried" + echo "case: a directory with no third-party actions passes with gh present" D="$TMP/nopins" workflow "$D" ci.yml <<EOF From bf8d4880b3f75acea3f71d562a318eb48f0269cd Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 11:28:01 +0000 Subject: [PATCH 084/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.234.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.234.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-plaud-direct-source.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-plaud-direct-source.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f03c8e78de..e434f84c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.234.0) + +### Added + +- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. + ## 2026-11-12 (v5.233.0) ### Added diff --git a/changelog.d/added-inbox-plaud-direct-source.md b/changelog.d/added-inbox-plaud-direct-source.md deleted file mode 100644 index 161055f469..0000000000 --- a/changelog.d/added-inbox-plaud-direct-source.md +++ /dev/null @@ -1 +0,0 @@ -- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. From 710bca77cc88dd21d6e01da97caf6913658a3042 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:48:12 +0100 Subject: [PATCH 085/223] =?UTF-8?q?=E2=9C=A8=20knowledge:=20prepare=20Wisp?= =?UTF-8?q?r=20Flow=20facts=20for=20Inbox=20(#17891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: prepare Wispr Flow facts for Inbox Hive-Run: hivecommons/hive#17878 Hive-Plan: voice-recorders-inbox Hive-Spec: voice-recorders-inbox#source Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> * chore: retrigger review after correcting issue link The PR description now references #17878 rather than closing it because the Source adapter remains outstanding in #17885. Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-wisprflow-inbox-metadata.md | 1 + src/docs/knowledge-connectors.md | 11 ++++--- src/pkg/knowledge/connector/wisprflow.go | 5 ++-- src/pkg/knowledge/connector/wisprflow_test.go | 30 ++++++++++++++++++- 4 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 changelog.d/added-wisprflow-inbox-metadata.md diff --git a/changelog.d/added-wisprflow-inbox-metadata.md b/changelog.d/added-wisprflow-inbox-metadata.md new file mode 100644 index 0000000000..70828adc48 --- /dev/null +++ b/changelog.d/added-wisprflow-inbox-metadata.md @@ -0,0 +1 @@ +- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index e5a24b695b..12dc561672 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -396,10 +396,13 @@ rest are still imported. seen minus a 2 h overlap (Flow updates a row in place as it finalises), or `since_days` back on the first run. With `group: day` the current day's fact is rebuilt whole. -- Facts carry `source: wisprflow`, `attr_wispr_transcript_id`, - `attr_dictated_at`, `attr_app` (bundle id), `attr_num_words`, - `attr_language` and the dictation date as the vault path - (`Wispr Flow/2026-11-11`). The title is the first ten words. +- Dictation facts carry `source: wisprflow`, `source_id`, + `attr_recorded_at`, `attr_wispr_transcript_id`, `attr_dictated_at`, + `attr_app` (bundle id), `attr_num_words`, `attr_language`, and the + dictation date as the vault path (`Wispr Flow/2026-11-11`). The title is + the first ten words, and the formatted text is under `## Transcript`. + With `group: day`, the fact represents the entire day and its + `attr_recorded_at` is the newest dictation's time. - The listing is incremental, so dictations you archive or delete in Flow are not tombstoned automatically; remove the fact file or the connector. - Everything here costs **$0** beyond your Wispr Flow plan; the free tier's diff --git a/src/pkg/knowledge/connector/wisprflow.go b/src/pkg/knowledge/connector/wisprflow.go index 39f902748e..9ac4fc239c 100644 --- a/src/pkg/knowledge/connector/wisprflow.go +++ b/src/pkg/knowledge/connector/wisprflow.go @@ -349,7 +349,7 @@ func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags s if r.Duration > 0 { fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(r.Duration * float64(time.Second))).Round(time.Second)) } - md.WriteString("\n") + md.WriteString("\n## Transcript\n\n") md.WriteString(r.Text) md.WriteString("\n") if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { @@ -360,6 +360,7 @@ func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags s attrs := map[string]string{ "wispr_transcript_id": r.ID, "dictated_at": r.At.UTC().Format(time.RFC3339), + "recorded_at": r.At.UTC().Format(time.RFC3339), } if r.App != "" { attrs["app"] = r.App @@ -405,7 +406,7 @@ func (c *wisprConnector) dayPage(day string, rows []wisprDictation, includeRaw b fmt.Fprintf(&md, "<details><summary>Raw transcript</summary>\n\n%s\n\n</details>\n\n", strings.TrimSpace(r.RawText)) } } - attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows))} + attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows)), "recorded_at": newest.UTC().Format(time.RFC3339)} if words > 0 { attrs["num_words"] = strconv.Itoa(words) } diff --git a/src/pkg/knowledge/connector/wisprflow_test.go b/src/pkg/knowledge/connector/wisprflow_test.go index c20df12fda..64545e2abd 100644 --- a/src/pkg/knowledge/connector/wisprflow_test.go +++ b/src/pkg/knowledge/connector/wisprflow_test.go @@ -195,6 +195,34 @@ func TestWisprSyncSQLite(t *testing.T) { } } +func TestWisprFactInboxMetadata(t *testing.T) { + c := newFakeWispr(t, map[string]string{"include_raw": "true"}) + pages, _, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + w := testWriter(t) + slug, _, err := w.Write(c.cfg, pages[1]) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(w.Dir, slug+".md")) + if err != nil { + t.Fatal(err) + } + fact := string(data) + for _, want := range []string{ + "source: wisprflow\n", + "source_id: d-2\n", + "attr_recorded_at: 2026-11-11T17:55:30Z\n", + "## Transcript\n\nRunning late — start without me.\n", + } { + if !strings.Contains(fact, want) { + t.Errorf("fact missing %q:\n%s", want, fact) + } + } +} + func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { c := newFakeWispr(t, map[string]string{"apps": "slack, vscode", "group": "day", "since_days": "3650"}) pages, _, err := collectWispr(t, c, "") @@ -205,7 +233,7 @@ func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { t.Fatalf("pages = %v", got) } day := pages[1] - if day.Attrs["dictations"] != "1" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { + if day.Attrs["dictations"] != "1" || day.Attrs["recorded_at"] != "2026-11-11T17:55:30Z" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { t.Errorf("day page = %v\n%s", day.Attrs, day.Markdown) } if strings.Contains(day.Markdown, "Ok.") { From 43ce61e558980564c739a4d051b5b60411332b94 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 11:59:18 +0000 Subject: [PATCH 086/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.234.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.234.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-check-action-pins-transient-api-error.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-check-action-pins-transient-api-error.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e434f84c45..6c6041a981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.234.1) + +### Fixed + +- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED <repo>@<sha> … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). + ## 2026-11-12 (v5.234.0) ### Added diff --git a/changelog.d/fixed-check-action-pins-transient-api-error.md b/changelog.d/fixed-check-action-pins-transient-api-error.md deleted file mode 100644 index b158778fd9..0000000000 --- a/changelog.d/fixed-check-action-pins-transient-api-error.md +++ /dev/null @@ -1 +0,0 @@ -- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED <repo>@<sha> … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). From fceea223710bfc8fb1771e73cdb4e974825b770a Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 12:28:38 +0000 Subject: [PATCH 087/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.235.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.235.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-wisprflow-inbox-metadata.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-wisprflow-inbox-metadata.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6041a981..0337292185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.235.0) + +### Added + +- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. + ## 2026-11-12 (v5.234.1) ### Fixed diff --git a/changelog.d/added-wisprflow-inbox-metadata.md b/changelog.d/added-wisprflow-inbox-metadata.md deleted file mode 100644 index 70828adc48..0000000000 --- a/changelog.d/added-wisprflow-inbox-metadata.md +++ /dev/null @@ -1 +0,0 @@ -- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. From 38430156615b160a147a8f0c172f681ecdc1c346 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:48:25 +0100 Subject: [PATCH 088/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20avoid=20premature?= =?UTF-8?q?=20eval-cycle=20test=20timeout=20(#17872)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: avoid premature eval-cycle test timeout Hive-Run: hivecommons/hive#17866 Hive-Plan: eval-cycle-flake Hive-Spec: ci-flake#17866 Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * test: isolate eval cycle from rate-limit pacing Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * ci: retry transient action pin resolution The reported missing hivecommons/infra commit resolves through the GitHub API; retry the check without changing the valid pin. Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/cmd/hive/eval_cycle_run_7232_test.go | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/cmd/hive/eval_cycle_run_7232_test.go b/src/cmd/hive/eval_cycle_run_7232_test.go index 8631532df0..ea07cd3dde 100644 --- a/src/cmd/hive/eval_cycle_run_7232_test.go +++ b/src/cmd/hive/eval_cycle_run_7232_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -25,10 +26,12 @@ import ( // touches on a quiet org: every list endpoint is empty, every object endpoint // is a minimal document, and nothing ever errors. It records the paths hit so // a test can assert the cycle actually enumerated rather than bailed early. -func newEvalCycleAPI(t *testing.T, hits *[]string) *httptest.Server { +func newEvalCycleAPI(t *testing.T, mu *sync.Mutex, hits *[]string) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() *hits = append(*hits, r.Method+" "+r.URL.Path) + mu.Unlock() w.Header().Set("Content-Type", "application/json") switch { case r.URL.Path == "/rate_limit": @@ -63,6 +66,7 @@ type evalCycleFixture struct { dashSrv *dashboard.Server last *atomic.Pointer[github.ActionableResult] hits []string + hitsMu sync.Mutex beadStores map[string]*beads.Store advisoryIssues map[string]int @@ -70,8 +74,11 @@ type evalCycleFixture struct { func newEvalCycleFixture(t *testing.T) *evalCycleFixture { t.Helper() + // A prior rate-limit test can leave process-wide request pacing enabled. + github.ResetRateLimitPacingForTest() + t.Cleanup(github.ResetRateLimitPacingForTest) f := &evalCycleFixture{last: new(atomic.Pointer[github.ActionableResult])} - api := newEvalCycleAPI(t, &f.hits) + api := newEvalCycleAPI(t, &f.hitsMu, &f.hits) logger := restoreTestLogger() f.cfg = &config.Config{} @@ -92,7 +99,8 @@ func newEvalCycleFixture(t *testing.T) *evalCycleFixture { func (f *evalCycleFixture) run(t *testing.T) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + // Bound a stalled in-process cycle without imposing a tight CI deadline. + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() runEvalCycle(ctx, f.cfg, f.gh, f.gov, f.sched, f.agentMgr, f.dashSrv, nil, // notifier @@ -128,7 +136,10 @@ func TestRunEvalCycle_QuietOrgEnumeratesAndPublishesStatus(t *testing.T) { f.run(t) - if len(f.hits) == 0 { + f.hitsMu.Lock() + hitCount := len(f.hits) + f.hitsMu.Unlock() + if hitCount == 0 { t.Fatal("runEvalCycle made no GitHub calls: it bailed before enumeration") } if got := f.last.Load(); got == nil { @@ -176,14 +187,17 @@ func TestRunEvalCycle_PostsAdvisoryDigestAndHealsAppAuthFinding(t *testing.T) { f.run(t) + f.hitsMu.Lock() + hits := append([]string(nil), f.hits...) + f.hitsMu.Unlock() var posted bool - for _, h := range f.hits { + for _, h := range hits { if h == "POST /repos/testorg/widget/issues/4/comments" { posted = true } } if !posted { - t.Fatalf("digest was never posted to the pinned issue; hits=%v", f.hits) + t.Fatalf("digest was never posted to the pinned issue; hits=%v", hits) } healed, err := store.Get(finding.ID) if err != nil { From fc9a59b4f713a13cfbecbcbd6a0ccb9c9583457b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:28:25 -0700 Subject: [PATCH 089/223] =?UTF-8?q?=F0=9F=90=9B=20gh-wrapper:=20let=20hive?= =?UTF-8?q?-baseline-check.sh's=20sibling=20lookup=20through=20the=20enume?= =?UTF-8?q?ration=20gate=20(#17838)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 gh-wrapper: let hive-baseline-check.sh's sibling lookup through the enumeration gate hive-baseline-check.sh is the kick's mandated triage step and its one narrow `gh pr list` projection is the sanctioned exception to the enumeration ban, but the wrapper refused it so the helper always exited 2. The helper now sets HIVE_BASELINE_CHECK=1 and the wrapper allows only --state open with --json statusCheckRollup and no --search; a policy block is reported distinctly from an API failure. Closes #17835 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 gh-wrapper: tighten _pr_list_baseline_check to an allowlist of the helper's exact projection Per maintainer review on #17838, the HIVE_BASELINE_CHECK=1 carve-out was wider than #17835's one projection: a marked `gh pr list --state open --label actionable --limit 500 --json number,title,labels,statusCheckRollup` passed. The predicate is now an allowlist that a forgotten flag cannot widen: only --repo/-R, --state/-s (open), --limit/-L (<=100) and --json are accepted, every --json field must be one of number,headRefOid,isCrossRepository,statusCheckRollup, and any other flag or positional refuses. The comment now states the marker is a caller-set hint, not a trust boundary. Tests: helper's exact call still allowed (plus -R/equals/short forms); --label, title field, --limit 500, missing --json and --state all blocked; existing --search and no-marker cases kept. Hive-Fix-Attempt: 1/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 gh-wrapper: validate every repeated --json in _pr_list_baseline_check gh accumulates repeated --json/--json= occurrences, but the baseline-check gate kept only the last value and validated that alone, so `--json title --json number` reached gh with `title` unchecked. Accumulate the fields across occurrences so the allowlist sees all of them, and add test rows for a blocked and an allowed repeated --json. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- bin/gh-wrapper.sh | 57 +++++++++++++++++++ bin/gh-wrapper.test.sh | 22 +++++++ bin/hive-baseline-check.sh | 12 +++- .../fixed-baseline-check-gh-wrapper.md | 1 + 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-baseline-check-gh-wrapper.md diff --git a/bin/gh-wrapper.sh b/bin/gh-wrapper.sh index 0284212fcc..b2c9d07624 100755 --- a/bin/gh-wrapper.sh +++ b/bin/gh-wrapper.sh @@ -592,6 +592,61 @@ _pr_list_merged_only() { [[ "$(_lower "$state")" = "merged" ]] } +# The one `gh pr list` that bin/hive-baseline-check.sh makes for its sibling-PR +# lookup (#17835): the kick mandates that helper before any check is retried, +# and names that lookup as the sanctioned exception to the enumeration ban. The +# helper marks its call with HIVE_BASELINE_CHECK=1. That marker is a caller-set +# hint, not a trust boundary — any agent can set it, exactly as with +# HIVE_CONTRIBUTOR_MODE above — so what makes the exception safe is that only +# the helper's exact projection is let through, checked as an allowlist that a +# forgotten flag cannot widen: the sole flags accepted are --repo/-R, --state/-s +# (must resolve to open), --limit/-L (at most 100, what the helper sends) and +# --json whose every field is one of number, headRefOid, isCrossRepository, +# statusCheckRollup. Any other flag or positional (--label, --search, --author, +# --base, -q, ...) refuses, since each would turn it into a general work queue. +_pr_list_baseline_check() { + [[ "${HIVE_BASELINE_CHECK:-}" = "1" ]] || return 1 + [[ "$subcmd" = "pr" && "$action" = "list" ]] || return 1 + local i a state="" json="" limit="" seen_sub=0 seen_act=0 + for ((i=0; i<${#args[@]}; i++)); do + a="${args[$i]}" + case "$a" in + --repo=*|-R=*|-R?*) ;; + --repo|-R) i=$((i+1)) ;; + --state=*) state="${a#--state=}" ;; + -s=*) state="${a#-s=}" ;; + -s?*) state="${a#-s}" ;; + --state|-s) i=$((i+1)); state="${args[$i]:-}" ;; + --limit=*) limit="${a#--limit=}" ;; + -L=*) limit="${a#-L=}" ;; + -L?*) limit="${a#-L}" ;; + --limit|-L) i=$((i+1)); limit="${args[$i]:-}" ;; + # gh accumulates repeated --json values, so every occurrence must be + # validated, not just the last one. + --json=*) json="${json:+${json},}${a#--json=}" ;; + --json) i=$((i+1)); json="${json:+${json},}${args[$i]:-}" ;; + -*) return 1 ;; + pr) [[ "$seen_sub" = 0 ]] || return 1; seen_sub=1 ;; + list) [[ "$seen_sub" = 1 && "$seen_act" = 0 ]] || return 1; seen_act=1 ;; + *) return 1 ;; + esac + done + [[ "$(_lower "$state")" = "open" ]] || return 1 + if [[ -n "$limit" ]]; then + [[ "$limit" =~ ^[0-9]+$ && "$limit" -le 100 ]] || return 1 + fi + [[ -n "$json" ]] || return 1 + local f _fields + IFS=',' read -ra _fields <<<"$json" + for f in "${_fields[@]}"; do + case "$f" in + number|headRefOid|isCrossRepository|statusCheckRollup) ;; + *) return 1 ;; + esac + done + return 0 +} + # Resolve the authenticated GitHub login. Initialize the cache internally so a # caller-controlled environment cannot seed a trusted identity (#3982). # @@ -726,6 +781,8 @@ if [ "$_gh_author_scoped" = true ]; then : elif _pr_list_merged_only; then : # Merged PRs only: history, not a work queue (see _pr_list_merged_only). + elif _pr_list_baseline_check; then + : # hive-baseline-check.sh's sibling lookup (see _pr_list_baseline_check). else # Root-caused in a live hive (2026-08-20): agents read this two-line # message as "all gh $subcmd commands are blocked" and silently skipped diff --git a/bin/gh-wrapper.test.sh b/bin/gh-wrapper.test.sh index 85d0a7175c..fd44b41572 100755 --- a/bin/gh-wrapper.test.sh +++ b/bin/gh-wrapper.test.sh @@ -495,6 +495,28 @@ _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state merged -S review (block pr list --repo test/repo --state merged -S review _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state merged --search=x (blocked: equals search flag)" \ pr list --repo test/repo --state merged --search=x +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list --state open statusCheckRollup with HIVE_BASELINE_CHECK=1 (allowed: baseline helper's sibling lookup)" \ + pr list --repo test/repo --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list -R before the subcommand, equals/short forms with HIVE_BASELINE_CHECK=1 (allowed: same projection)" \ + -R test/repo pr list --state=open -L50 --json=number,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open --search with HIVE_BASELINE_CHECK=1 (blocked)" \ + pr list --repo test/repo --state open --search foo +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --label actionable with HIVE_BASELINE_CHECK=1 (blocked: flag outside the allowlist)" \ + pr list --repo test/repo --state open --label actionable --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --json number,title,statusCheckRollup with HIVE_BASELINE_CHECK=1 (blocked: field outside the projection)" \ + pr list --repo test/repo --state open --limit 100 --json number,title,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --json title --json number with HIVE_BASELINE_CHECK=1 (blocked: gh accumulates repeated --json, every occurrence is checked)" \ + pr list --repo test/repo --state open --limit 100 --json title --json number +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list --json number --json statusCheckRollup with HIVE_BASELINE_CHECK=1 (allowed: repeated --json, all fields in the projection)" \ + pr list --repo test/repo --state open --limit 100 --json number --json statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --limit 500 with HIVE_BASELINE_CHECK=1 (blocked: over the helper's 100)" \ + pr list --repo test/repo --state open --limit 500 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open without --json with HIVE_BASELINE_CHECK=1 (blocked: no projection)" \ + pr list --repo test/repo --state open --limit 100 +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state all with HIVE_BASELINE_CHECK=1 (blocked: --state open required)" \ + pr list --repo test/repo --state all --limit 100 --json number,statusCheckRollup +_run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open --json statusCheckRollup without HIVE_BASELINE_CHECK (blocked)" \ + pr list --repo test/repo --state open --json statusCheckRollup _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state with no value (blocked)" \ pr list --repo test/repo --state _run_test_staff_gate 1 "$ENUM_DENY" "-" "issue list --state merged (blocked: issues are never merged)" \ diff --git a/bin/hive-baseline-check.sh b/bin/hive-baseline-check.sh index 83b930aa8f..07756ca84d 100755 --- a/bin/hive-baseline-check.sh +++ b/bin/hive-baseline-check.sh @@ -155,8 +155,16 @@ if (( BASE_AGE_HOURS > MAX_AGE_HOURS )); then BASE_STALE=true fi -if ! PRS_JSON="$(gh pr list --repo "$REPO" --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup)"; then - error "could not read open PR checks for $REPO" +# HIVE_BASELINE_CHECK=1 marks this call for the gh wrapper, which lets only this +# narrow projection through its `gh pr list` enumeration gate (#17835). +PRS_ERR_FILE="$(mktemp)" +trap 'rm -f "$PRS_ERR_FILE"' EXIT +if ! PRS_JSON="$(HIVE_BASELINE_CHECK=1 gh pr list --repo "$REPO" --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup 2>"$PRS_ERR_FILE")"; then + if grep -qE 'enumeration_disabled|BLOCKED' "$PRS_ERR_FILE"; then + error "gh wrapper policy blocked the open PR lookup for $REPO (not a GitHub API failure): $(head -c 300 "$PRS_ERR_FILE" | tr '\n' ' ')" + fi + cat "$PRS_ERR_FILE" >&2 + error "could not read open PR checks for $REPO (GitHub API failure)" fi if ! SIBLING_PRS="$(jq -cer --arg check "$CHECK_NAME" ' [ diff --git a/changelog.d/fixed-baseline-check-gh-wrapper.md b/changelog.d/fixed-baseline-check-gh-wrapper.md new file mode 100644 index 0000000000..4d3ccfe69b --- /dev/null +++ b/changelog.d/fixed-baseline-check-gh-wrapper.md @@ -0,0 +1 @@ +- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. From 76f9e9e414a14dff75ab73e01191a5229e3777e6 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:28:30 -0700 Subject: [PATCH 090/223] =?UTF-8?q?=E2=9C=A8=20[v5]=20claude=20auth:=20run?= =?UTF-8?q?=20Claude=20agents=20on=20an=20operator-supplied=20long-lived?= =?UTF-8?q?=20token=20(HIVE=5FCLAUDE=5FOAUTH=5FTOKEN)=20(#17890)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ claude auth: run Claude agents on an operator-supplied long-lived token (HIVE_CLAUDE_OAUTH_TOKEN) (#17762) * ✨ claude auth: inject operator-supplied long-lived token via HIVE_CLAUDE_OAUTH_TOKEN Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> * 🐛 claude auth: honor HIVE_CLAUDE_OAUTH_TOKEN in headless launches; resync api-reference citations HeadlessCredentialEnv now applies the same precedence as agentEnvPairs: the operator-supplied long-lived token wins over the short-lived credentials snapshot, so hub-launched claude CLIs no longer pin to a token that cannot refresh. Adds a unit test for the new path. api-reference.md citations for GET /api/logs/sources and /api/logs/read resynced via check-api-reference-citations.sh --fix after the import added in api_logs.go shifted the registrations by one line. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🔒 claude auth: strip HIVE_CLAUDE_OAUTH_TOKEN from every agent pane's inherited env The hive process holds HIVE_CLAUDE_OAUTH_TOKEN in its own environment, and each tmux server it starts inherits that into its GLOBAL environment, which tmux copies into every pane it forks. applySessionEnv never removed it, so copilot/codex/bob agent shells (and every tool shell they spawn) could read the year-long token under its original name — contradicting the claim that the token never appears in a pane. The claude backend was unaffected in practice only because it also receives the value as CLAUDE_CODE_OAUTH_TOKEN. Add claude.OperatorTokenEnvVar to inheritedCredentialEnvVars so the creation-time strip records a `set-environment -r` removal for it, hiding the global value from every process forked afterwards. The file form (HIVE_CLAUDE_OAUTH_TOKEN_FILE) carries only a path and is unaffected. The real-tmux leak test now plants the variable in the server env and asserts a fresh pane no longer sees it; a tmux-free unit test pins the entry in inheritedCredentialEnvVars. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: retrigger build after buildx GHA cache BlobNotFound Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 8c35975f2d2cf371656e9e2c64a79e0db31397da) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: explain Claude setup-token operator flow (#17841) * 📖 docs: explain Claude setup-token operator flow Hive-Run: hivecommons/hive#17761 Hive-Plan: 17753 item 8 documentation Hive-Spec: 17753#docs Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> * chore: correct PR closing reference Remove the unrelated closing reference from the PR description; this signed follow-up requests a fresh review. Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> (cherry picked from commit 7f27bbaa3859cdeb22811537d69ce24644b729fc) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 claude auth: suspend the credential re-bridge and durable-credential watchdog while HIVE_CLAUDE_OAUTH_TOKEN is set (#17858) With the operator's long-lived token set, Claude agents authenticate through CLAUDE_CODE_OAUTH_TOKEN and the shared ~/.claude/.credentials.json is not in use. The per-agent credential re-bridge (launch, watchdog poll, post-sign-in) and the watchdog's Claude probe now do nothing in that mode: no files moved, no claude_token_missing audit or warning, no broken-bridge flag in status. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset nothing changes. Fixes #17760 Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Andrew Anderson <andy@clubanderson.com> (cherry picked from commit 9903fea75b994e36310fc97c1623f5d40e77c5cb) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 fix(claude): adapt long-lived token port to v5 v5 has no HeadlessCredentialEnv, no /api/logs read API and no CredentialBridgeBroken status field, so those parts of the v6 commits have nothing to attach to here: the headless and log-scrub hunks were dropped during the #17762 cherry-pick, and the manager-level status override (snapshotAgent) from #17858 is removed. The operator-token-mode tests now build on v5's bridgeTestFixture/simulateClaudeRefresh helpers. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-claude-long-lived-token.md | 1 + ...claude-operator-token-suspends-rebridge.md | 1 + src/crux/agent/claude_credential_bridge.go | 15 +- src/crux/agent/claude_operator_token_mode.go | 38 ++++ .../agent/claude_operator_token_mode_test.go | 127 +++++++++++++ src/crux/agent/claude_operator_token_test.go | 167 ++++++++++++++++++ src/crux/agent/credentials_watchdog.go | 11 +- src/crux/agent/manager.go | 17 +- src/crux/agent/manager_env.go | 10 +- src/crux/agent/manager_tmux.go | 8 + src/crux/agent/tmux_inherited_env_test.go | 3 +- src/crux/claude/operator_token.go | 40 +++++ src/crux/claude/operator_token_test.go | 41 +++++ src/docs/agent-configuration.md | 26 +++ src/docs/backend-auth-recovery-runbook.md | 18 +- src/docs/env-vars.md | 4 +- 16 files changed, 514 insertions(+), 13 deletions(-) create mode 100644 changelog.d/added-claude-long-lived-token.md create mode 100644 changelog.d/changed-claude-operator-token-suspends-rebridge.md create mode 100644 src/crux/agent/claude_operator_token_mode.go create mode 100644 src/crux/agent/claude_operator_token_mode_test.go create mode 100644 src/crux/agent/claude_operator_token_test.go create mode 100644 src/crux/claude/operator_token.go create mode 100644 src/crux/claude/operator_token_test.go diff --git a/changelog.d/added-claude-long-lived-token.md b/changelog.d/added-claude-long-lived-token.md new file mode 100644 index 0000000000..9c84edba5b --- /dev/null +++ b/changelog.d/added-claude-long-lived-token.md @@ -0,0 +1 @@ +- Claude agents can run on an operator-supplied long-lived token: set `HIVE_CLAUDE_OAUTH_TOKEN` (or mount it at `/secrets/claude-oauth-token`, or point `HIVE_CLAUDE_OAUTH_TOKEN_FILE` at it) to a `claude setup-token` token and every `claude`-backend agent is launched with it as `CLAUDE_CODE_OAUTH_TOKEN`, ahead of the 8-hour dashboard sign-in; unset, nothing changes. diff --git a/changelog.d/changed-claude-operator-token-suspends-rebridge.md b/changelog.d/changed-claude-operator-token-suspends-rebridge.md new file mode 100644 index 0000000000..22c625b98c --- /dev/null +++ b/changelog.d/changed-claude-operator-token-suspends-rebridge.md @@ -0,0 +1 @@ +- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go index b511e8d955..1f36cb2096 100644 --- a/src/crux/agent/claude_credential_bridge.go +++ b/src/crux/agent/claude_credential_bridge.go @@ -521,6 +521,14 @@ func preserveClaudeCredential(path string, now time.Time) (string, error) { // outcome. It is the single entry point the launch path, the watchdog poll and // the sign-in hook share, so all three log and audit identically. func (m *Manager) rebridgeClaudeCredential(agentName, link, target, trigger string) *claudeBridgeRepair { + // Operator-token mode: agents do not authenticate with the shared + // credentials file, so a private copy left by a refresh is nothing to + // repair (claude_operator_token_mode.go). Covers launch, watchdog and + // sign-in alike. + if m.claudeOperatorTokenMode() { + m.logClaudeOperatorTokenModeOnce() + return nil + } rep, err := m.repairClaudeCredentialBridge(agentName, link, target, trigger) if errors.Is(err, errClaudeBridgeLinkIsTarget) { // The legacy layout: skipped, and said once per agent rather than on @@ -594,8 +602,13 @@ func (m *Manager) logClaudeBridgeUnreadable(agentName, link, trigger string, err // RepairClaudeCredentialBridges checks every per-agent Claude credential // bridge and repairs each one a regular file has replaced. It returns the // names of the agents it re-bridged, sorted. Agents are snapshotted under m.mu -// and repaired after it is released. +// and repaired after it is released. In operator-token mode it does nothing +// and returns nil (claude_operator_token_mode.go). func (m *Manager) RepairClaudeCredentialBridges(trigger string) []string { + if m.claudeOperatorTokenMode() { + m.logClaudeOperatorTokenModeOnce() + return nil + } type candidate struct { name string uid int diff --git a/src/crux/agent/claude_operator_token_mode.go b/src/crux/agent/claude_operator_token_mode.go new file mode 100644 index 0000000000..76898bd3ea --- /dev/null +++ b/src/crux/agent/claude_operator_token_mode.go @@ -0,0 +1,38 @@ +package agent + +// Operator-token mode for Claude (#17760). +// +// With HIVE_CLAUDE_OAUTH_TOKEN set, every claude-backend agent runs on the +// operator's long-lived `claude setup-token` token through +// CLAUDE_CODE_OAUTH_TOKEN (manager_env.go). That token is never refreshed, so +// the machinery built around the shared ~/.claude/.credentials.json — the +// per-agent credential re-bridge that publishes a refreshed token back into +// the shared login, and the durable-credential watchdog that alerts when that +// file is missing or spent — has nothing to act on: the file is not what the +// agents authenticate with. Acting anyway would move credential files around +// for no reason; alerting would send an operator to redo a dashboard sign-in +// that the fleet does not use. Both are therefore suspended for Claude while +// the token is set and behave exactly as before when it is unset. The token's +// own health (a 401 from the API) is surfaced separately (#17759). + +// claudeOperatorTokenMode reports whether Claude agents run on the operator's +// long-lived token rather than the shared credentials file. The token is read +// once at construction (claudeOperatorToken), so the mode is fixed for the +// life of the process. +func (m *Manager) claudeOperatorTokenMode() bool { + return m.claudeOperatorToken != "" +} + +// logClaudeOperatorTokenModeOnce says, once per process, that the Claude +// credential re-bridge and watchdog alerts are suspended, so an operator +// reading the log knows why neither will ever fire. Never logs the token. +func (m *Manager) logClaudeOperatorTokenModeOnce() { + if !m.claudeOperatorTokenMode() { + return + } + m.claudeOperatorModeLogOnce.Do(func() { + m.logger.Info("claude auth: operator-supplied long-lived token is set; Claude agents run on it, so the shared credentials file is not in use — credential re-bridge and durable-credential watchdog alerts are suspended for the claude backend", + "source", "HIVE_CLAUDE_OAUTH_TOKEN", + "suspended", "claude credential re-bridge, claude durable-credential watchdog") + }) +} diff --git a/src/crux/agent/claude_operator_token_mode_test.go b/src/crux/agent/claude_operator_token_mode_test.go new file mode 100644 index 0000000000..e34eb37464 --- /dev/null +++ b/src/crux/agent/claude_operator_token_mode_test.go @@ -0,0 +1,127 @@ +package agent + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// operatorModeFixture is a manager in operator-token mode (the token field set +// as NewManager does from HIVE_CLAUDE_OAUTH_TOKEN) with one claude agent whose +// credential link a refresh replaced, plus the log it writes. +func operatorModeFixture(t *testing.T) (*Manager, *fakeAuditSink, *AgentProcess, string, string, *bytes.Buffer) { + t.Helper() + m, sink, ap, link, target := bridgeTestFixture(t) + var logs bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + m.claudeOperatorToken = operatorTokenSentinel + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "shared", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "private", now.Add(8*time.Hour).UnixMilli())) + return m, sink, ap, link, target, &logs +} + +// While the operator token is set the re-bridge does nothing on any trigger: +// the private file stays a private file, the shared login is untouched, +// and nothing is audited. +func TestClaudeOperatorTokenMode_RebridgeIsSuspended(t *testing.T) { + m, sink, _, link, target, logs := operatorModeFixture(t) + sharedBefore, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + agentBefore, err := os.ReadFile(link) + if err != nil { + t.Fatal(err) + } + + for _, trigger := range []string{claudeBridgeTriggerWatchdog, claudeBridgeTriggerSignIn} { + if got := m.RepairClaudeCredentialBridges(trigger); len(got) != 0 { + t.Fatalf("trigger %s: RepairClaudeCredentialBridges = %v, want none in operator-token mode", trigger, got) + } + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerLaunch); rep != nil { + t.Fatalf("launch re-bridge ran in operator-token mode: %+v", rep) + } + m.rebridgeClaudeCredentialsAfterSignIn() + + if fi, err := os.Lstat(link); err != nil || !fi.Mode().IsRegular() { + t.Fatalf("agent's private credential must be left exactly where it is; Lstat: %v %v", fi, err) + } + if got, _ := os.ReadFile(link); !bytes.Equal(got, agentBefore) { + t.Error("agent's private credential was rewritten") + } + if got, _ := os.ReadFile(target); !bytes.Equal(got, sharedBefore) { + t.Error("shared credential was rewritten") + } + if stale, _ := filepath.Glob(filepath.Join(filepath.Dir(link), "*"+claudeCredentialStaleInfix+"*")); len(stale) != 0 { + t.Errorf("files were moved aside: %v", stale) + } + if n := sink.count(); n != 0 { + t.Errorf("re-bridge audited %d entries in operator-token mode, want 0", n) + } + if n := strings.Count(logs.String(), "watchdog alerts are suspended"); n != 1 { + t.Errorf("suspension logged %d times across several calls, want exactly once:\n%s", n, logs.String()) + } + if strings.Contains(logs.String(), operatorTokenSentinel) { + t.Fatal("operator token reached the log") + } +} + +// While the operator token is set the watchdog never alerts about the Claude +// credentials file — absent, spent or otherwise — but still watches Copilot. +func TestClaudeOperatorTokenMode_WatchdogClaudeProbeIsSuspended(t *testing.T) { + m, sink, _, _, _, logs := operatorModeFixture(t) + m.mu.Lock() + m.agents["reviewer"] = &AgentProcess{Name: "reviewer", UID: 2002, Config: config.AgentConfig{Backend: "copilot"}} + m.mu.Unlock() + + missing := filepath.Join(t.TempDir(), "missing") + last := map[string]bool{"claude": true} // a stale observation from before token mode is cleared, not re-alerted + m.evalCredentialWatch(credentialWatch{backend: "claude", path: missing, + auditAction: AuditClaudeTokenMissing, probe: claudeTokenUsable}, last) + if sink.count() != 0 { + t.Fatalf("claude watch audited %d entries in operator-token mode, want 0", sink.count()) + } + if _, tracked := last["claude"]; tracked { + t.Error("suspended claude watch must not keep a transition record") + } + if strings.Contains(logs.String(), "durable credential unusable") { + t.Errorf("claude watch warned in operator-token mode:\n%s", logs.String()) + } + + m.evalCredentialWatch(credentialWatch{backend: "copilot", path: missing, + auditAction: AuditCopilotTokenMissing, probe: copilotTokenUsable}, last) + if e := sink.find(AuditCopilotTokenMissing); e == nil { + t.Fatal("copilot watch must keep alerting in operator-token mode") + } +} + +// Unset, the same fixture behaves exactly as today: the re-bridge repairs the +// link and the watchdog probes Claude. +func TestClaudeOperatorTokenMode_UnsetKeepsTodaysBehaviour(t *testing.T) { + m, sink, _, link, target, logs := operatorModeFixture(t) + m.claudeOperatorToken = "" + + if got := m.RepairClaudeCredentialBridges(claudeBridgeTriggerWatchdog); len(got) != 1 || got[0] != "scanner" { + t.Fatalf("RepairClaudeCredentialBridges = %v, want [scanner]", got) + } + assertLinkRestored(t, link, target) + last := map[string]bool{} + m.evalCredentialWatch(credentialWatch{backend: "claude", path: filepath.Join(t.TempDir(), "missing"), + auditAction: AuditClaudeTokenMissing, probe: claudeTokenUsable}, last) + if e := sink.find(AuditClaudeTokenMissing); e == nil { + t.Fatal("claude watch must alert when no operator token is set") + } + if strings.Contains(logs.String(), "watchdog alerts are suspended") { + t.Error("suspension must not be logged when the token is unset") + } +} diff --git a/src/crux/agent/claude_operator_token_test.go b/src/crux/agent/claude_operator_token_test.go new file mode 100644 index 0000000000..8ad2b15647 --- /dev/null +++ b/src/crux/agent/claude_operator_token_test.go @@ -0,0 +1,167 @@ +package agent + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" +) + +const operatorTokenSentinel = "sk-ant-oat01-OPERATOR-SENTINEL-7f3c9a1e5b" + +// stageOperatorToken makes claude.OperatorToken hermetic: the env var holds +// token ("" for unset) and the default secret file points at a missing path. +func stageOperatorToken(t *testing.T, token string) { + t.Helper() + t.Setenv(claude.OperatorTokenEnvVar, token) + t.Setenv(claude.OperatorTokenFileEnvVar, "") + orig := claude.OperatorTokenFile + claude.OperatorTokenFile = filepath.Join(t.TempDir(), "missing") + t.Cleanup(func() { claude.OperatorTokenFile = orig }) +} + +// With a long-lived operator token set, an agent whose credential file is +// readable and fresh still runs on the operator token: that file is the +// 8-hour login that logs the fleet out (#17753). +func TestClaudeOperatorToken_InjectedOverAReadableCredential(t *testing.T) { + stageSharedClaudeCredential(t, map[string]any{ + "accessToken": "sk-ant-oat-live", + "expiresAt": time.Now().Add(6 * time.Hour).UnixMilli(), + }) + stageOperatorToken(t, operatorTokenSentinel) + m, agent := claudeAgentFixture(t, "sk-ant-oat-snapshot-from-boot") + + pair, ok := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"] + if !ok || pair.Value != operatorTokenSentinel { + t.Fatalf("pair = %+v, %v; want the operator token over a readable credential", pair, ok) + } + if !pair.Secret { + t.Fatal("the operator token must be a secret pair (tmux set-environment, never a command line)") + } +} + +// With no credential file at all, the operator token is still injected, and it +// beats the dashboard snapshot the last-resort fallback would otherwise use. +func TestClaudeOperatorToken_InjectedWithNoCredentialFile(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, operatorTokenSentinel) + m, agent := claudeAgentFixture(t, "sk-ant-oat-from-dashboard-login") + + pair, ok := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"] + if !ok || pair.Value != operatorTokenSentinel || !pair.Secret { + t.Fatalf("pair = %+v, %v; want the operator token as a secret pair", pair, ok) + } +} + +// A dashboard sign-in after boot only replaces claudeAuthToken; it must not +// switch agents back to the 8-hour token. +func TestClaudeOperatorToken_WinsOverALaterDashboardSignIn(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, operatorTokenSentinel) + m, agent := claudeAgentFixture(t, "") + m.mu.Lock() + m.claudeAuthToken = "sk-ant-oat-fresh-dashboard-login" + m.mu.Unlock() + + pairs := m.agentEnvPairs(agent) + var got []string + for _, p := range pairs { + if p.Key == "CLAUDE_CODE_OAUTH_TOKEN" { + got = append(got, p.Value) + } + } + if len(got) != 1 || got[0] != operatorTokenSentinel { + t.Fatalf("CLAUDE_CODE_OAUTH_TOKEN values = %q; want exactly the operator token", got) + } +} + +// Unset: behaviour is exactly today's — no override over a readable +// credential, the dashboard snapshot when nothing is readable. +func TestClaudeOperatorToken_UnsetKeepsTodaysBehaviour(t *testing.T) { + stageSharedClaudeCredential(t, map[string]any{ + "accessToken": "sk-ant-oat-live", + "expiresAt": time.Now().Add(6 * time.Hour).UnixMilli(), + }) + stageOperatorToken(t, "") + m, agent := claudeAgentFixture(t, "sk-ant-oat-snapshot-from-boot") + if _, ok := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"]; ok { + t.Fatal("with the operator token unset, nothing may be injected over a readable credential") + } + + stageSharedClaudeCredential(t, nil) + m, agent = claudeAgentFixture(t, "sk-ant-oat-from-dashboard-login") + if pair := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"]; pair.Value != "sk-ant-oat-from-dashboard-login" { + t.Fatalf("with the operator token unset, the last-resort fallback must be unchanged: %+v", pair) + } +} + +// The token can come from the secret-file form and is never sent to an agent +// on another backend. +func TestClaudeOperatorToken_FileFormAndOtherBackends(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, "") + path := filepath.Join(t.TempDir(), "claude-oauth-token") + if err := os.WriteFile(path, []byte(operatorTokenSentinel+"\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(claude.OperatorTokenFileEnvVar, path) + + m, agent := claudeAgentFixture(t, "") + if pair := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"]; pair.Value != operatorTokenSentinel { + t.Fatalf("file form not injected: %+v", pair) + } + + other := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "copilot"}, + }, discardLogger(), ProjectContext{ACMMLevel: 5}) + other.mu.RLock() + otherAgent := other.agents["scanner"] + other.mu.RUnlock() + if _, ok := envPairsByKey(other, otherAgent)["CLAUDE_CODE_OAUTH_TOKEN"]; ok { + t.Fatal("a copilot agent must not receive the operator Claude token") + } +} + +// The manager never writes the operator token to its log while loading it +// and building an agent's environment. +func TestClaudeOperatorToken_NeverLogged(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, operatorTokenSentinel) + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + m := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "claude", Model: "claude-sonnet-5"}, + }, logger, ProjectContext{ACMMLevel: 5}) + m.mu.RLock() + agent := m.agents["scanner"] + m.mu.RUnlock() + _ = m.agentEnvPairs(agent) + _ = m.buildEnvPrefix(agent) + + if strings.Contains(buf.String(), operatorTokenSentinel) { + t.Fatalf("operator token written to the hive log: %s", buf.String()) + } + if strings.Contains(m.buildEnvPrefix(agent), operatorTokenSentinel) { + t.Fatal("operator token reached the shell command-line env prefix") + } +} + +// The hive process holds HIVE_CLAUDE_OAUTH_TOKEN in its own environment, and +// every tmux server it starts inherits that globally. applySessionEnv must +// strip it (with -r, so the global value is hidden too) or every pane of every +// backend — copilot, codex, bob — could read the year-long token under its +// original name. The claude backend receives it only as CLAUDE_CODE_OAUTH_TOKEN. +func TestClaudeOperatorToken_StrippedFromInheritedPaneEnv(t *testing.T) { + for _, k := range inheritedCredentialEnvVars { + if k == claude.OperatorTokenEnvVar { + return + } + } + t.Fatalf("%s is not in inheritedCredentialEnvVars; it would reach every agent pane via the tmux global environment", claude.OperatorTokenEnvVar) +} diff --git a/src/crux/agent/credentials_watchdog.go b/src/crux/agent/credentials_watchdog.go index 5a3c02d85e..7259d488e0 100644 --- a/src/crux/agent/credentials_watchdog.go +++ b/src/crux/agent/credentials_watchdog.go @@ -220,7 +220,13 @@ func credentialWatches() []credentialWatch { // Safe to start unconditionally at boot: like StartAgentTokenRefresh it tracks // live state each tick rather than a boot-time snapshot, so a hive that gains // or loses a backend via config reload is evaluated correctly. +// +// Operator-token mode (HIVE_CLAUDE_OAUTH_TOKEN set): the Claude agents do not +// use the shared credentials file, so the Claude re-bridge and the Claude +// probe are suspended and said so once at start; Copilot is watched as +// before. See claude_operator_token_mode.go. func (m *Manager) StartCredentialWatchdog(ctx context.Context) { + m.logClaudeOperatorTokenModeOnce() ticker := time.NewTicker(credentialWatchdogInterval()) defer ticker.Stop() bridgeTicker := time.NewTicker(claudeBridgeInterval()) @@ -254,8 +260,9 @@ func (m *Manager) StartCredentialWatchdog(ctx context.Context) { // use) and emits a transition-edge log + audit event. lastUnusable carries the // previous observation per backend across ticks. func (m *Manager) evalCredentialWatch(w credentialWatch, lastUnusable map[string]bool) { - if !m.backendInUse(w.backend) { - // Not in use: nothing to watch. Reset the tracker so a later config + if !m.backendInUse(w.backend) || (w.backend == "claude" && m.claudeOperatorTokenMode()) { + // Not in use (or, for Claude, the operator token is in use instead of + // the file): nothing to watch. Reset the tracker so a later config // change that adds this backend with a bad credential alerts on its // first miss rather than being masked as "no transition". delete(lastUnusable, w.backend) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 92102c3b9d..9713d21661 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -594,12 +594,16 @@ type Manager struct { // precedence for whatever fresh token the operator supplied. copilotAuthTokenRejected bool claudeAuthToken string - uidMap *UIDMap - sharedPool SharedSessionPool - uidMapPath string - appAuth AppTokenMinter - appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged - agentMint AgentMintIssuer // optional, opt-in mint credential (nil ⇒ off) + claudeOperatorToken string // HIVE_CLAUDE_OAUTH_TOKEN, read once at construction; a dashboard sign-in never replaces it + // claudeOperatorModeLogOnce guards the one-time "re-bridge and watchdog + // suspended" log for operator-token mode (claude_operator_token_mode.go). + claudeOperatorModeLogOnce sync.Once + uidMap *UIDMap + sharedPool SharedSessionPool + uidMapPath string + appAuth AppTokenMinter + appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged + agentMint AgentMintIssuer // optional, opt-in mint credential (nil ⇒ off) // sharedPoolRenewInterval overrides the lease watcher tick (zero means // defaultSharedPoolRenewInterval). Set before any watcher starts; it is @@ -886,6 +890,7 @@ func NewManagerWithOptions(agents map[string]config.AgentConfig, logger *slog.Lo copilotAuthTokenAuthoritative: copilotTokenAuthoritative, copilotAuthTokenSource: copilotTokenSource, claudeAuthToken: claudeToken, + claudeOperatorToken: claude.OperatorToken(), uidMap: uidMap, sharedPool: opts.sharedPool, uidMapPath: selectedUIDMapPath, diff --git a/src/crux/agent/manager_env.go b/src/crux/agent/manager_env.go index c930d652aa..9e866de3d2 100644 --- a/src/crux/agent/manager_env.go +++ b/src/crux/agent/manager_env.go @@ -422,7 +422,15 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // keeps the variable doing the job it was added for (#c5648bc9: deliver a // dashboard-obtained token to an agent that cannot see the file) and stops // it overriding a credential that can still refresh itself. - if m.claudeAuthToken != "" && backend == "claude" && !claudeCredentialReachable(agent, backend) { + // + // An operator-supplied long-lived token (HIVE_CLAUDE_OAUTH_TOKEN, from + // `claude setup-token`) is the exception: it lasts about a year and never + // needs a refresh, so pinning it is the point. It is injected whether or + // not a credentials file is reachable and wins over the dashboard + // sign-in, which only ever updates claudeAuthToken. + if m.claudeOperatorToken != "" && backend == "claude" { + vars = append(vars, agentEnvPair{"CLAUDE_CODE_OAUTH_TOKEN", m.claudeOperatorToken, true}) + } else if m.claudeAuthToken != "" && backend == "claude" && !claudeCredentialReachable(agent, backend) { vars = append(vars, agentEnvPair{"CLAUDE_CODE_OAUTH_TOKEN", m.claudeAuthToken, true}) } // bob reads its key from BOBSHELL_API_KEY in 1.x and BOB_API_KEY in 2.x. diff --git a/src/crux/agent/manager_tmux.go b/src/crux/agent/manager_tmux.go index 87932ded5e..4358f9b333 100644 --- a/src/crux/agent/manager_tmux.go +++ b/src/crux/agent/manager_tmux.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hivecommons/hive/crux/agent/tmux" + "github.com/hivecommons/hive/crux/claude" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/internal/textutil" "github.com/hivecommons/hive/crux/upstreamproxy" @@ -50,8 +51,15 @@ var defaultTmuxSocket string // global value from every process forked afterwards; the sanctioned // per-agent pair set afterwards overrides the removal for the one variable // the agent is meant to have. +// +// HIVE_CLAUDE_OAUTH_TOKEN is the operator's long-lived Claude token. It is +// delivered to claude-backend agents only, as CLAUDE_CODE_OAUTH_TOKEN via +// agentEnvPairs; under its own name it must reach no pane at all, or every +// copilot/codex/bob shell (and every tool shell they spawn) could read a +// year-long credential it has no use for. var inheritedCredentialEnvVars = []string{ "HIVE_GITHUB_TOKEN", + claude.OperatorTokenEnvVar, linearAPIKeyEnvVar, linearAccessTokenEnvVar, "LINEAR_CLIENT_ID", diff --git a/src/crux/agent/tmux_inherited_env_test.go b/src/crux/agent/tmux_inherited_env_test.go index 2b8a40b799..bf9821249a 100644 --- a/src/crux/agent/tmux_inherited_env_test.go +++ b/src/crux/agent/tmux_inherited_env_test.go @@ -62,6 +62,7 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { "HIVE_UPSTREAM_PROXY_HOSTS=ghe.corp.example", "HIVE_UPSTREAM_PROXY_NO_PROXY=10.0.0.0/8", "HIVE_NO_PROXY=10.0.0.0/8", + "HIVE_CLAUDE_OAUTH_TOKEN=sk-ant-oat01-operator_leaked", ) if out, err := start.CombinedOutput(); err != nil { t.Fatalf("starting tmux server: %v: %s", err, out) @@ -126,7 +127,7 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { "LINEAR_API_KEY", "LINEAR_CLIENT_SECRET", "LINEAR_WEBHOOK_SECRET", "HIVE_GITHUB_TOKEN", "GITHUB_TOKEN", "DASHBOARD_AUTH_TOKEN", "HIVE_DASHBOARD_TOKEN", "HIVE_TERMINAL_KEY", "HIVE_INVITE_KEY", "HIVE_HEARTBEAT_KEY", "HIVE_HUB_SECRET", "HIVE_TTYD_CREDENTIAL", "HIVE_UPSTREAM_PROXY_URL", "HIVE_UPSTREAM_PROXY_HOSTS", - "HIVE_UPSTREAM_PROXY_NO_PROXY", "HIVE_NO_PROXY", + "HIVE_UPSTREAM_PROXY_NO_PROXY", "HIVE_NO_PROXY", "HIVE_CLAUDE_OAUTH_TOKEN", } { if v, ok := after[k]; ok { t.Errorf("%s=%q still reaches a pane after applySessionEnv — the inherited credential was not removed", k, v) diff --git a/src/crux/claude/operator_token.go b/src/crux/claude/operator_token.go new file mode 100644 index 0000000000..f0f7bb3b00 --- /dev/null +++ b/src/crux/claude/operator_token.go @@ -0,0 +1,40 @@ +package claude + +import ( + "os" + "strings" +) + +const ( + // OperatorTokenEnvVar holds an operator-generated long-lived token + // (`claude setup-token`). When set, every claude-backend agent runs on it + // through CLAUDE_CODE_OAUTH_TOKEN instead of the dashboard sign-in. + OperatorTokenEnvVar = "HIVE_CLAUDE_OAUTH_TOKEN" + + // OperatorTokenFileEnvVar names a file holding the same token, for + // deployments that mount it as a secret rather than an env var. + OperatorTokenFileEnvVar = "HIVE_CLAUDE_OAUTH_TOKEN_FILE" +) + +// OperatorTokenFile is the mounted Secret key read when neither variable is +// set, mirroring the dashboard token's /secrets/dashboard-token. A var so tests +// can point it at a hermetic path. +var OperatorTokenFile = "/secrets/claude-oauth-token" + +// OperatorToken returns the operator-supplied long-lived Claude token, or "" +// when none is configured. The env var wins over the file. The value is a +// secret: callers must only hand it to an agent as a secret env pair. +func OperatorToken() string { + if v := strings.TrimSpace(os.Getenv(OperatorTokenEnvVar)); v != "" { + return v + } + path := strings.TrimSpace(os.Getenv(OperatorTokenFileEnvVar)) + if path == "" { + path = OperatorTokenFile + } + data, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} diff --git a/src/crux/claude/operator_token_test.go b/src/crux/claude/operator_token_test.go new file mode 100644 index 0000000000..ec3e5223ef --- /dev/null +++ b/src/crux/claude/operator_token_test.go @@ -0,0 +1,41 @@ +package claude + +import ( + "os" + "path/filepath" + "testing" +) + +func TestOperatorTokenSources(t *testing.T) { + dir := t.TempDir() + orig := OperatorTokenFile + OperatorTokenFile = filepath.Join(dir, "default-token") + t.Cleanup(func() { OperatorTokenFile = orig }) + t.Setenv(OperatorTokenEnvVar, "") + t.Setenv(OperatorTokenFileEnvVar, "") + + if got := OperatorToken(); got != "" { + t.Fatalf("unset: got %q, want empty", got) + } + + if err := os.WriteFile(OperatorTokenFile, []byte(" sk-ant-oat01-default \n"), 0o600); err != nil { + t.Fatal(err) + } + if got := OperatorToken(); got != "sk-ant-oat01-default" { + t.Fatalf("default secret file: got %q", got) + } + + named := filepath.Join(dir, "named-token") + if err := os.WriteFile(named, []byte("sk-ant-oat01-named\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(OperatorTokenFileEnvVar, named) + if got := OperatorToken(); got != "sk-ant-oat01-named" { + t.Fatalf("%s: got %q", OperatorTokenFileEnvVar, got) + } + + t.Setenv(OperatorTokenEnvVar, " sk-ant-oat01-env ") + if got := OperatorToken(); got != "sk-ant-oat01-env" { + t.Fatalf("env must win over the file: got %q", got) + } +} diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index e270605f8b..49a8f6bba0 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -157,6 +157,32 @@ The dashboard API can update this field through [`POST /api/effort/{agent}/{effo The dashboard shows a reasoning-effort dropdown next to the model picker for every method that has an effort control (`codex`, `agy`, `claude`, and `omp`). The value is validated at set time against the method's own set, so a `codex`-only level such as `minimal` is refused for a `claude` agent rather than stored and dropped at launch. For `omp`, the dropdown is per selected model: Hive reads `omp models --json` and uses that model's `thinking` levels, hiding the control for models that report no thinking support. Picking `default` clears the stored value; for `claude` and `omp` that means no effort flag at all and the CLI's own default. The dropdown persists through the same path as the model picker (`hive.yaml` plus the per-agent overlay under `agents_dir`), which is the durable path on a hosted spoke: a hand edit of the agent's file is reverted by the next sync, the dashboard write is not. Each dropdown change is recorded in the audit trail as `set_reasoning_effort` with the new value, like the other per-agent setting changes. The effort the agent was actually launched with is what the run rows and the `- hive:` PR trailer report, so a `claude` agent at `xhigh` shows `effort=xhigh` there and an agent with no stored effort shows none. +#### Claude setup-token + +For `backend: claude`, an operator can use a long-lived token instead of the +dashboard sign-in. Run `claude setup-token` interactively as the licensed +operator and complete the prompts. The resulting token lasts about a year and +is inference-only; it is not a general-purpose account credential. See +[Claude Code authentication](https://code.claude.com/docs/en/authentication). + +Supply it to the Hive process as the secret `HIVE_CLAUDE_OAUTH_TOKEN`, or mount +it in a secret file readable by Hive. Set `HIVE_CLAUDE_OAUTH_TOKEN_FILE` to +that file's path, or use the default `/secrets/claude-oauth-token`. The +non-empty environment variable wins over the file. Do not put the token in +`hive.yaml`, shell history, logs, issues, or agent panes. + +Hive passes the token to Claude agents as `CLAUDE_CODE_OAUTH_TOKEN` at launch, +even when a credentials file exists. It takes precedence over dashboard sign-in; +signing in again on the dashboard does not replace it. Each agent retains its +own Claude home and private state; sharing the token does not require sharing +the whole home. Without an operator token, dashboard sign-in works as before. + +To replace an expired or revoked token, run `claude setup-token` again, update +the configured secret source, then recreate or restart Hive to load the new +token and restart the affected Claude agents so their new sessions receive +the replacement. Verify an agent completes a turn. See the +[backend auth recovery runbook](backend-auth-recovery-runbook.md). + #### Oh My Pi (`omp`) hub agents `backend: omp` launches the OMP CLI as a first-class dashboard method. Hive gives each agent its own durable HOME under `/data/home/agents/<name>`, so `~/.omp/agent/agent.db` can hold several provider sign-ins for that one agent (for example `openai-codex` for the primary model and `anthropic` for an advisor) without sharing refresh-token writers across agents. Select models with their provider prefix (`provider/model`). The dashboard model probe runs `omp models --json` and uses each model's `thinking` list for the effort dropdown. At launch Hive passes `--approval-mode ${HIVE_OMP_APPROVAL_MODE:-yolo}`, `--model <id>`, and `--thinking <effort>` when set. diff --git a/src/docs/backend-auth-recovery-runbook.md b/src/docs/backend-auth-recovery-runbook.md index 5045b55ac1..76a710ea0c 100644 --- a/src/docs/backend-auth-recovery-runbook.md +++ b/src/docs/backend-auth-recovery-runbook.md @@ -51,11 +51,27 @@ recovered and no action is needed. | `backendAuthStatus` | First action | |---|---| | `unlicensed` | Check the seat at <https://github.com/settings/copilot>. If it is active, the hive is presenting a token the account no longer owns: follow the `/login` recovery in [troubleshooting.md](troubleshooting.md). | -| `token-expired` | If `backendAuthRefreshable` is `true` (provider `auth_state: refreshable`), restart the affected agents: Claude Code redeems the refresh grant on start and no login is needed. Otherwise run `/login` in any agent's Terminal with a licensed identity and wait about 30 seconds for the reconciler; affected agents relaunch onto the new token. | +| `token-expired` | For Claude with an operator setup token, replace it as described below; dashboard `/login` does not override it. Otherwise, if `backendAuthRefreshable` is `true` (provider `auth_state: refreshable`), restart the affected agents so Claude Code redeems the refresh grant. If not, run `/login` in any agent's Terminal with a licensed identity and wait about 30 seconds for the reconciler. | | `forbidden` | Cause is undetermined (bare 403). Read which credential was refused (dashboard model dropdown hover, or the `copilot model discovery rejected by upstream` log line's `credential` field) before re-logging in; an org policy denial is not fixed by a new login. | | `unreachable` | The inference backend is down or unroutable. Check the backend's own health and network path from the pod; re-login will not help. | | `quota` | The provider is refusing calls on quota. Wait for the window to reset, add quota, or switch the agents to another backend. Do not restart agents in a loop. | +### Replace a Claude operator token + +If `HIVE_CLAUDE_OAUTH_TOKEN` or its secret file is configured, a rejected +token may have expired or been revoked. Run `claude setup-token` interactively +as the licensed operator and complete its prompts. The replacement lasts about +a year and is inference-only. Store it securely in the same Hive environment +secret or in the file named by `HIVE_CLAUDE_OAUTH_TOKEN_FILE` (default +`/secrets/claude-oauth-token`); the non-empty environment variable takes +precedence over the file. Never paste the token into a pane, log, or report. + +Recreate or restart Hive to load the changed secret, then restart affected +Claude agents to give their new sessions the replacement. Follow the +verification steps below. A dashboard sign-in does not supersede the configured +operator token. See [Claude setup-token](agent-configuration.md#claude-setup-token) +for initial setup. + ## 3. Verify recovery 1. Within one reconciler tick (about 30 seconds) the affected agents drop diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index fd410f8a76..8f317217ad 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -295,6 +295,8 @@ and [network/credential controls](security-model.md#5-network-and-credentials). | `HIVE_BOB_API_URL` | No | `https://api.us-east.bob.ibm.com` | Bob key-test endpoint base URL override. | | `BOBSHELL_API_KEY` | Required by Bob CLI when Hive injects or contributor mode uses Bob | none | API key name read by bobshell itself. | | `COPILOT_GITHUB_TOKEN` | No | dashboard device-flow token file, if present | Copilot completion/model-discovery token and explicit agent injection. | +| `HIVE_CLAUDE_OAUTH_TOKEN` | No | none | Hive-side secret from `claude setup-token`; takes precedence over the secret file and dashboard sign-in for Claude agents. See [Claude setup-token](agent-configuration.md#claude-setup-token). | +| `HIVE_CLAUDE_OAUTH_TOKEN_FILE` | No | `/secrets/claude-oauth-token` | Path to a file containing the setup token, used when `HIVE_CLAUDE_OAUTH_TOKEN` is empty. Keep the file readable by Hive and restricted as a secret. | | `ANTHROPIC_API_KEY` | Required by `cmd/apiproxy` to inject an upstream key; agent inference backends receive a synthetic value | none | Anthropic-compatible **upstream** API key. Never used to authenticate callers of `cmd/apiproxy`. | | `PROXY_AUTH_TOKEN` | **Required** by `cmd/apiproxy`; the binary exits at startup when unset and the proxy handler returns `503` | none | **Client auth token** callers must present to `cmd/apiproxy` via `Authorization: Bearer` or `X-Api-Key`. Validated in constant time and stripped before the upstream request. Mandatory because an unauthenticated proxy would let any co-resident loopback caller spend the host `ANTHROPIC_API_KEY`. | | `CONTEXT7_API_KEY` | No | none | Optional key for Context7 knowledge API integration. | @@ -386,7 +388,7 @@ Set conditionally: | `COPILOT_GITHUB_TOKEN` | When the hive holds a Copilot auth token | Copilot OAuth token (authenticates the AI model, not GitHub writes). **Secret.** | | `GITHUB_TOKEN` | Only when App-authored PRs are enabled (`AppAuthoredPRs`), an App is configured, and the agent's mode can push | The per-agent tier-scoped App installation token, so the built-in GitHub MCP server writes as the App bot. Advisory agents have `GITHUB_TOKEN` deliberately stripped. **Secret.** | | `LINEAR_ACCESS_TOKEN` / `LINEAR_API_KEY` | ISSUES_ONLY+ agents on Linear-connected hives | See the [Linear agent integration](#linear-agent-integration) note above. | -| `CLAUDE_CODE_OAUTH_TOKEN` | Last resort: claude backend **and** no readable credential file | Dashboard-obtained access token; never injected when the agent can read (and refresh) `~/.claude/.credentials.json` itself. **Secret.** | +| `CLAUDE_CODE_OAUTH_TOKEN` | Claude backend with an operator token, or as a last resort when no credential file is readable | Hive injects the operator's setup token when configured, even if a credential file exists. Otherwise the fallback is the dashboard-obtained access token only when no credential file is readable. **Secret.** | | `BOBSHELL_API_KEY`, `BOB_API_KEY`, `BOBSHELL_DEFAULT_AUTH_TYPE` | bob backend only | The resolved Bob API key for Bob 1.x / 2.x (**secret**) and the literal `api-key` auth-type selector (non-secret by design - see the #2228 relaunch note in code). | | `HIVE_BOB_SESSION_LABEL` | bob backend only when `governor.bob.session_prefix` or `agents.<name>.bob.session_label` is set | The Bob session label also passed to Bob as `--instance-id`. A label only: `HIVE_AGENT` and all functional Hive identity are unchanged. See [agent-configuration.md](agent-configuration.md). | | `HIVE_BOB_DISPLAY_NAME` | bob backend only | Legacy alias for the Bob session label, falling back to the agent name for older scripts. Prefer `HIVE_BOB_SESSION_LABEL` when present. | From b9623477636c54603b3f2dec56c4479a7187e940 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:32:34 -0700 Subject: [PATCH 091/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20hub-signed=20rela?= =?UTF-8?q?y=20lets=20hosted=20spokes=20verify=20relayed=20GitHub=20webhoo?= =?UTF-8?q?ks=20(#17914)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: hub-signed relay lets hosted spokes verify relayed GitHub webhooks Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> * 📝 docs: refresh api-reference citations shifted by relay-key change Running check-api-reference-citations.sh --fix after the hub relay-key code landed above the cited route registrations in server.go and api_review_event_dispatch.go. The api_review_event_dispatch.go:16 -> :15 citation uses the "GET "+constant shape the fixer cannot rewrite, so it is corrected by hand. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🔒 fix: store hive meta.json with the relay key rw------- like other hub secrets SaaSHive.WebhookRelayKey is serialised into saasHivesDir/<id>/meta.json, which saveSaaSHive wrote 0o644 under a 0o755 directory. The record carried no key material before this PR; the hub's other secret stores (cluster App keys, generation stores, session revocation) and the spoke-side relay key file all use 0o600/0o700. A locally leaked relay key would let an attacker forge deliveries to that hive's public /api/webhook/github. Write meta.json 0o600, create the hive directory 0o700, and best-effort chmod pre-existing directories so hives provisioned before this change are tightened on their next save. Add a regression test covering both modes. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 fix: tighten hive dir perms without widening a read-only dir saveSaaSHive forced the hive directory to 0700, which re-added owner-write to directories the reset-assignment tests lock to 0555 to simulate a failed persist, so TestHandleResetAssignmentReportsSaveFailure and TestSweepStuckAssignmentsSkipsWedgeWhoseSaveFails no longer saw the failure. Strip only group/other bits so legacy 0755 dirs still become 0700 while a deliberately read-only dir keeps failing the write. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * ♻️ test: reuse useTempSaaSHivesDir in saveSaaSHive perm tests TestSaveSaaSHiveKeepsRelayKeyPrivate and TestSaveSaaSHiveNeverWidensReadOnlyDir swapped saasHivesDir for t.TempDir() inline even though the file already defines useTempSaaSHivesDir for exactly that. Use the helper like the sibling tests do. Hive-Fix-Attempt: 3/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * chore: retrigger merge watcher after GitHub 503 exhausted the prior merge request Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-hosted-webhook-relay-signature.md | 1 + src/cmd/hive/main.go | 12 + src/docs/api-reference/agents-and-controls.md | 2 +- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/env-vars.md | 2 +- .../operator-reference/runtime-and-quota.md | 33 +- src/pkg/dashboard/api_github_webhook.go | 29 +- .../dashboard/api_review_event_dispatch.go | 8 +- src/pkg/dashboard/server.go | 8 +- src/pkg/dashboard/webhook_relay.go | 128 ++++++++ src/pkg/dashboard/webhook_relay_test.go | 268 ++++++++++++++++ src/pkg/github/webhook_cache.go | 6 +- src/pkg/github/webhook_relay_key.go | 134 ++++++++ src/pkg/github/webhook_relay_key_test.go | 70 +++++ src/pkg/hub/heartbeat.go | 5 + src/pkg/hub/lite_enrollment.go | 1 + src/pkg/hub/pool_replenisher.go | 7 +- src/pkg/hub/saas_hives_api.go | 11 +- src/pkg/hub/saas_provision.go | 26 +- src/pkg/hub/server.go | 15 + src/pkg/hub/spoke/heartbeat.go | 16 + src/pkg/hub/webhook.go | 16 +- src/pkg/hub/webhook_relay_key.go | 121 ++++++++ src/pkg/hub/webhook_relay_key_test.go | 286 ++++++++++++++++++ 27 files changed, 1178 insertions(+), 59 deletions(-) create mode 100644 changelog.d/fixed-hosted-webhook-relay-signature.md create mode 100644 src/pkg/dashboard/webhook_relay.go create mode 100644 src/pkg/dashboard/webhook_relay_test.go create mode 100644 src/pkg/github/webhook_relay_key.go create mode 100644 src/pkg/github/webhook_relay_key_test.go create mode 100644 src/pkg/hub/webhook_relay_key.go create mode 100644 src/pkg/hub/webhook_relay_key_test.go diff --git a/changelog.d/fixed-hosted-webhook-relay-signature.md b/changelog.d/fixed-hosted-webhook-relay-signature.md new file mode 100644 index 0000000000..436f4aa770 --- /dev/null +++ b/changelog.d/fixed-hosted-webhook-relay-signature.md @@ -0,0 +1 @@ +- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 2cbba7bd59..4e643473d5 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4852,6 +4852,9 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // nothing and a spoke holding the wrong one self-heals. GitHubAppKeyFingerprint: spoke.TimedSection("app_key_fingerprint", func() string { return appKeys.ReportedFingerprint(b.cfg.GitHub.KeyFile, b.cfg.GitHub.AppID) }), GitHubAppKeyPerHive: spoke.TimedSection("app_key_per_hive", func() bool { return appKeys.HasPerHiveKey(b.cfg.GitHub.KeyFile, b.cfg.GitHub.AppID) }), + // Fingerprint only (#17911): the hub re-delivers the webhook + // relay key until this matches the key on its record. + WebhookRelayKeyFingerprint: github.WebhookRelayKeyFingerprint(), // Report the App this hive believes it authenticates as. The hub // pairs it with the fingerprint above to tell a per-hive key that // is WRONG for this App from one that is deliberately for another. @@ -5228,6 +5231,15 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { "has_key", ghCfg.PrivateKey != "", ) + // The webhook relay key (#17911) is independent of the App + // identity below, so it is adopted before the identity guard can + // refuse the rest of the delivery. Empty never blanks a held key. + if changed, err := github.AdoptWebhookRelayKey(ghCfg.WebhookRelayKey); err != nil { + b.logger.Error("webhook relay key adopted in memory but not persisted", "error", err) + } else if changed { + b.logger.Info("adopted webhook relay key from hub", "fingerprint", github.WebhookRelayKeyFingerprint()) + } + // WRITE-PATH GUARD. Compute the identity this push WOULD produce and // refuse the whole delivery if it is internally inconsistent. // diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index bf60205f3f..7f42dc9717 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -27,7 +27,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:518` | | `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:519` | | `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:523` | -| `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:16` | +| `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | | `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:527` | | `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:528` | | `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:530` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 8c5b76d503..1d3e6c8eff 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1687` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1688` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1689` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1688` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1689` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1690` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index aec304411e..8511ded0f0 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1694` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1695` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index a0737fea7e..85fa9acf7b 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1667` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1668` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1669` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1675` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1680` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1682` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1683` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1684` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1685` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1686` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1668` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1669` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1670` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1676` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1682` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1683` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1684` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1685` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1686` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1687` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 96caa5bfa3..c3da74dff5 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1757` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1758` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 8f317217ad..23d80031c7 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -104,7 +104,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTE_SKIP_LABELS` | No | `blocked,tracking,epic,discussion,question,needs-decision,needs-triage` | Comma-separated, case-insensitive label patterns for issues that are not contributor work and must never be offered by the relay. Patterns use `path.Match`-style `*` globs; `blocked` is always unioned into the effective set even if omitted. Same setting as `hub.contribute_skip_labels`. | | `HIVE_FEDERATION_REGISTRY_PATH` | No | `/data/federation/registry.json` | Federation registry path override. | | `HIVE_WEBHOOK_SECRET` | No | none | HMAC secret for the spoke `/webhook` channel. | -| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret and rejects every delivery while it is unset. | +| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a self-hosted spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret. A hosted spoke leaves it unset and verifies hub relays with its own per-hive relay key from the heartbeat; with neither, every delivery is rejected. | | `HIVE_DASHBOARD_URL` | No | none | Base URL the `hive tui` client targets (`pkg/tui/client`). A bad value surfaces as a request error on the first call, not at startup. On the hub it is also consulted (fourth) in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_CONVERGENCE_MODE` | No | `convergence.mode` in `hive.yaml`, else `shadow` | Process-level override of the convergence mode (`off`, `shadow`, `enforce`) so an operator can flip the mode without editing `hive.yaml`. An **unset** mode takes the default, `shadow`, which computes and records admission decisions without withholding any work. A **non-empty but unrecognised** value — a typo, or a mode this build does not know — still resolves to `off`. | | `HIVE_RELEASE_LINE_LAG_MAX` | No | `5` | Alarm threshold for the release-line drift surface ([#6960](https://github.com/hivecommons/hive/issues/6960)): how many commits the hosted edge line (`v5`) may sit behind the stable default branch (`v4`) before the dashboard status flags the line as drifting (`releaseLineLag.exceeded`). An unset, empty, non-numeric, or negative value falls back to the default — it can never silently disable the alarm. An unknown lag (tips unresolved or the compare failed) is reported as `unknown`, never a healthy zero. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index f85565eff8..bd85fb4553 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -262,18 +262,18 @@ accuracy guarantees of the ForgeState contract. A spoke accepts GitHub App webhooks on the public `POST /api/webhook/github`, either delivered directly by GitHub or relayed by the hub (the hub forwards the original signed body to the hive that manages the repository). The receiver -fails closed: it rejects every delivery until `GITHUB_WEBHOOK_SECRET` is set to -the App's webhook secret, and it verifies `X-Hub-Signature-256` over the raw -body exactly as the hub does. Deliveries for repositories the hive does not -manage are ignored. +fails closed: it accepts a delivery only when `X-Hub-Signature-256` verifies +against `GITHUB_WEBHOOK_SECRET` (exactly as the hub does), or when the hub's +relay signature verifies against this hive's own relay key (hosted spokes, +below). With neither configured it rejects every delivery with 503. +Deliveries for repositories the hive does not manage are ignored. #### What the operator configures -1. **Secret.** Set `GITHUB_WEBHOOK_SECRET` in the spoke's environment (a - Secret mount on Kubernetes; never in `hive.yaml`) to the App's webhook - secret. On a hosted spoke this is the same value as the hub's - `webhook-secret.key`. Without it the receiver answers 503 and the loop - stays on the polling cadence. +1. **Secret (self-hosted only).** Set `GITHUB_WEBHOOK_SECRET` in the spoke's + environment (a Secret mount on Kubernetes; never in `hive.yaml`) to the + App's webhook secret. A hosted spoke needs nothing here: it never holds the + App secret and verifies relays with its hub relay key instead (item 3). 2. **GitHub App webhook.** Point the App's webhook URL at the hub (`POST /api/github/webhook`) for hosted spokes, or at the spoke's own `/api/webhook/github` for a stand-alone hive (a repository or organization @@ -286,8 +286,14 @@ manage are ignored. (organization-scoped; see [Work sources](../work-sources.md#type-github_projects--github-projects-projects-v2)). 3. **Hub relay.** The hub verifies the HMAC and relays the body byte-for-byte, with the original signature and `X-GitHub-Delivery`, to the spoke whose - reported dashboard URL manages the repository. Nothing to configure on the - spoke beyond the shared secret. + reported dashboard URL manages the repository. For a hosted hive it also + adds `X-Hive-Relay-Timestamp` and `X-Hive-Relay-Signature-256` + (`sha256=` HMAC-SHA256 over `timestamp.delivery-id.body`) keyed with that + hive's own relay key, which the hub mints at provisioning and delivers over + the heartbeat (persisted on the spoke as `/data/webhook-relay.key`). The + spoke rejects a relay whose signature does not verify, whose timestamp is + more than 5 minutes off, or whose delivery ID it already accepted. + Nothing to configure. 4. **Pacing (optional).** `governor.eval_interval_webhook_s` (default `900`) is the base tick while webhooks are healthy; `governor.webhook_wake_debounce_s` (default `10`) and `governor.webhook_wake_min_gap_s` (default `30`) pace the @@ -675,9 +681,10 @@ starve the governor: `/api/status` and `/api/gh-rate-limits` expose `webhooks: {healthy, last_event_at, events_1h, invalidations_1h, targeted_refreshes_1h, tracker_invalidations_1h, mode, since, mode_reason, -healthy_windows}` (the middle two count the single-item re-reads and the +healthy_windows, verification}` (the middle two count the single-item re-reads and the tracker cache drops webhooks caused; `mode` is `webhook` | `polling` and -`since` when it began — see [webhooks.md](../webhooks.md)); +`since` when it began — see [webhooks.md](../webhooks.md); `verification` is +`github-signed`, `relay (hub-signed)` or `none`); `/api/status` adds `evalMode` (`webhook` | `poll`), `evalIntervalS`, `lastWakeReason`, `lastWakeAt` and `webhookWakes`; `graphql_pr_batch.webhook_skips` counts batch queries skipped because nothing diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index b8ad183a36..539de1084a 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -20,10 +20,11 @@ import ( const ( // githubWebhookPath is the PUBLIC path GitHub (or the hub relay) posts App // webhooks to. Not actually open: the handler fails closed without - // GITHUB_WEBHOOK_SECRET and verifies X-Hub-Signature-256 over the raw body. + // GITHUB_WEBHOOK_SECRET or a hub relay key, and verifies + // X-Hub-Signature-256 or the hub relay signature over the raw body. githubWebhookPath = "/api/webhook/github" - githubWebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + githubWebhookSecretEnvVar = github.WebhookSecretEnvVar githubWebhookMaxBodyBytes = 1 << 20 ) @@ -61,7 +62,10 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { return } secret := strings.TrimSpace(os.Getenv(githubWebhookSecretEnvVar)) - if secret == "" { + // A hosted spoke holds no App webhook secret; it verifies the hub's relay + // signature with this hive's own relay key instead (#17911). + relayKey := github.WebhookRelayKey() + if secret == "" && relayKey == "" { const msg = "github webhook rejected: no webhook secret configured (set GITHUB_WEBHOOK_SECRET to enable signed webhooks)" warned := false s.webhookNoSecretOnce.Do(func() { @@ -74,11 +78,20 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { jsonError(w, "webhooks disabled: no secret configured", http.StatusServiceUnavailable) return } - if !verifyGitHubWebhookSignature(body, r.Header.Get("X-Hub-Signature-256"), secret) { - s.webhookLogger().Warn("github webhook signature verification failed", "event", event) - github.RecordWebhookDeliveryFailure() - jsonError(w, "invalid signature", http.StatusUnauthorized) - return + if secret == "" || !verifyGitHubWebhookSignature(body, r.Header.Get("X-Hub-Signature-256"), secret) { + if relayKey == "" || r.Header.Get(webhookRelaySignatureHeader) == "" { + s.webhookLogger().Warn("github webhook signature verification failed", "event", event) + github.RecordWebhookDeliveryFailure() + jsonError(w, "invalid signature", http.StatusUnauthorized) + return + } + if reason := s.verifyWebhookRelay(r.Header, body, relayKey, time.Now()); reason != "" { + s.webhookLogger().Warn("hub-relayed github webhook rejected", "event", event, "reason", reason, + "delivery", r.Header.Get("X-GitHub-Delivery")) + github.RecordWebhookDeliveryFailure() + jsonError(w, "relay rejected: "+reason, http.StatusUnauthorized) + return + } } if event == githubWebhookEventProjectsV2Item { // Organization-scoped: no repository to resolve, so it never reaches diff --git a/src/pkg/dashboard/api_review_event_dispatch.go b/src/pkg/dashboard/api_review_event_dispatch.go index 1ce5817d56..6484e91e25 100644 --- a/src/pkg/dashboard/api_review_event_dispatch.go +++ b/src/pkg/dashboard/api_review_event_dispatch.go @@ -3,9 +3,8 @@ package dashboard import ( "encoding/json" "net/http" - "os" - "strings" + "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/review/eventdispatch" "github.com/hivecommons/hive/pkg/review/pipeline" ) @@ -24,8 +23,11 @@ type reviewEventDispatchResponse struct { eventdispatch.Status } +// githubWebhooksConfigured reports whether this spoke can verify webhook +// deliveries at all: with GITHUB_WEBHOOK_SECRET, or (hosted) with the hub +// relay key delivered over the heartbeat (#17911). func githubWebhooksConfigured() bool { - return strings.TrimSpace(os.Getenv(githubWebhookSecretEnvVar)) != "" + return github.WebhookVerification() != github.WebhookVerificationNone } func (s *Server) reviewEvents() *eventdispatch.Dispatcher { diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index e1db31882f..77b80d02e8 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -106,7 +106,8 @@ const staleDropStarvationThreshold = 5 type Server struct { nativeQueueTracker nativeQueueTrackerRuntime integration integrationRuntime - webhookNoSecretOnce sync.Once // warn once per process about a missing webhook secret (#13666) + webhookNoSecretOnce sync.Once // warn once per process about a missing webhook secret (#13666) + webhookRelaySeen webhookRelayReplayCache // hub-relayed delivery IDs already accepted (#17911) feedbackRefreshBackoff feedbackBackoff @@ -2494,8 +2495,9 @@ func (s *Server) isPublicPath(path string) bool { case path == githubWebhookPath: // GitHub App webhooks (direct or relayed by the hub): GitHub cannot // hold a dashboard session. NOT actually open — the handler fails - // closed without GITHUB_WEBHOOK_SECRET and verifies the HMAC - // X-Hub-Signature-256 over the raw body. + // closed without GITHUB_WEBHOOK_SECRET or a hub relay key and + // verifies the HMAC X-Hub-Signature-256 (or the hub's relay + // signature, with replay protection) over the raw body. return true case path == linearAgentWebhookPath: // Linear AgentSessionEvent webhooks: Linear's servers cannot hold a diff --git a/src/pkg/dashboard/webhook_relay.go b/src/pkg/dashboard/webhook_relay.go new file mode 100644 index 0000000000..24f01987c0 --- /dev/null +++ b/src/pkg/dashboard/webhook_relay.go @@ -0,0 +1,128 @@ +package dashboard + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// Hub relay signature on relayed GitHub webhooks (hivecommons/hive#17911). +// +// A hosted spoke never holds the GitHub App webhook secret, so the hub signs +// each delivery it relays with this hive's own relay key (delivered over the +// heartbeat; github.WebhookRelayKey). The scheme mirrors pkg/hub +// webhookRelaySignature: +// +// X-Hive-Relay-Timestamp: <unix seconds at relay time> +// X-Hive-Relay-Signature-256: sha256=<hex HMAC-SHA256(key, timestamp + "." + X-GitHub-Delivery + "." + raw body)> +// +// After the MAC verifies, a timestamp more than webhookRelayMaxSkew from now +// is rejected as stale, and a delivery ID already accepted inside the replay +// window is rejected as replayed. +const ( + webhookRelaySignatureHeader = "X-Hive-Relay-Signature-256" + webhookRelayTimestampHeader = "X-Hive-Relay-Timestamp" + + webhookRelayMaxSkew = 5 * time.Minute + // webhookRelayReplayTTL covers every timestamp still inside the skew + // window, so a captured relay cannot be replayed while it would verify. + webhookRelayReplayTTL = 2 * webhookRelayMaxSkew + // webhookRelayReplayMax bounds the remembered delivery IDs; the oldest is + // dropped when a burst exceeds it. + webhookRelayReplayMax = 4096 +) + +// Rejection reasons for hub-relayed deliveries, logged and returned in the +// 401 body; each is counted as a failed delivery like a bad GitHub signature. +const ( + webhookRelayReasonBadSignature = "relay_signature_invalid" + webhookRelayReasonNoDelivery = "relay_delivery_missing" + webhookRelayReasonStale = "relay_timestamp_stale" + webhookRelayReasonReplayed = "relay_delivery_replayed" +) + +// verifyWebhookRelaySignature is the constant-time HMAC-SHA256 check of the +// relay signature, in the same style as verifyGitHubWebhookSignature. +func verifyWebhookRelaySignature(body []byte, timestamp, delivery, signature, key string) bool { + if key == "" || !strings.HasPrefix(signature, "sha256=") { + return false + } + sig, err := hex.DecodeString(strings.TrimPrefix(signature, "sha256=")) + if err != nil { + return false + } + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(timestamp)) + mac.Write([]byte(".")) + mac.Write([]byte(delivery)) + mac.Write([]byte(".")) + mac.Write(body) + return hmac.Equal(sig, mac.Sum(nil)) +} + +// webhookRelayReplayCache remembers recently accepted relay delivery IDs. +// The zero value is ready to use. +type webhookRelayReplayCache struct { + mu sync.Mutex + seen map[string]time.Time +} + +// firstSeen records id at now and reports whether it was new. Expired +// entries are pruned on the way in. +func (c *webhookRelayReplayCache) firstSeen(id string, now time.Time) bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.seen == nil { + c.seen = make(map[string]time.Time) + } + if at, ok := c.seen[id]; ok && now.Sub(at) < webhookRelayReplayTTL { + return false + } + for k, at := range c.seen { + if now.Sub(at) >= webhookRelayReplayTTL { + delete(c.seen, k) + } + } + if len(c.seen) >= webhookRelayReplayMax { + oldestID, oldestAt := "", now + for k, at := range c.seen { + if oldestID == "" || at.Before(oldestAt) { + oldestID, oldestAt = k, at + } + } + delete(c.seen, oldestID) + } + c.seen[id] = now + return true +} + +// verifyWebhookRelay checks a hub-relayed delivery and returns "" when it is +// accepted, otherwise the rejection reason. The delivery ID is remembered +// only once the signature and timestamp verify, so unsigned traffic cannot +// fill the replay cache. +func (s *Server) verifyWebhookRelay(h http.Header, body []byte, key string, now time.Time) string { + ts := h.Get(webhookRelayTimestampHeader) + delivery := h.Get("X-GitHub-Delivery") + if !verifyWebhookRelaySignature(body, ts, delivery, h.Get(webhookRelaySignatureHeader), key) { + return webhookRelayReasonBadSignature + } + if delivery == "" { + return webhookRelayReasonNoDelivery + } + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return webhookRelayReasonStale + } + if skew := now.Sub(time.Unix(sec, 0)); skew > webhookRelayMaxSkew || skew < -webhookRelayMaxSkew { + return webhookRelayReasonStale + } + if !s.webhookRelaySeen.firstSeen(delivery, now) { + return webhookRelayReasonReplayed + } + return "" +} diff --git a/src/pkg/dashboard/webhook_relay_test.go b/src/pkg/dashboard/webhook_relay_test.go new file mode 100644 index 0000000000..fca1551aa8 --- /dev/null +++ b/src/pkg/dashboard/webhook_relay_test.go @@ -0,0 +1,268 @@ +package dashboard + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// signWebhookRelayForTest is the hub's relay signature (pkg/hub +// webhookRelaySignature), computed independently here. +func signWebhookRelayForTest(key, ts, delivery string, body []byte) string { + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(ts + "." + delivery + ".")) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +// withWebhookRelayKey gives this spoke a hub relay key (as a heartbeat +// delivery would) in a temp store, and no GITHUB_WEBHOOK_SECRET. +func withWebhookRelayKey(t *testing.T, key string) { + t.Helper() + t.Setenv(githubWebhookSecretEnvVar, "") + t.Cleanup(github.SetWebhookRelayKeyPathForTest(filepath.Join(t.TempDir(), "webhook-relay.key"))) + if key == "" { + return + } + if changed, err := github.AdoptWebhookRelayKey(key); err != nil || !changed { + t.Fatalf("adopt relay key: changed=%v err=%v", changed, err) + } +} + +func postRelayedWebhook(s *Server, event, delivery, ts, relaySig string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, githubWebhookPath, bytes.NewReader(body)) + req.Header.Set("X-GitHub-Event", event) + if delivery != "" { + req.Header.Set("X-GitHub-Delivery", delivery) + } + // GitHub's own signature rides along unchanged; this spoke cannot verify it. + req.Header.Set("X-Hub-Signature-256", signGitHubWebhookForTest("app-secret-the-spoke-never-has", body)) + if ts != "" { + req.Header.Set(webhookRelayTimestampHeader, ts) + } + if relaySig != "" { + req.Header.Set(webhookRelaySignatureHeader, relaySig) + } + rec := httptest.NewRecorder() + s.handleGitHubWebhook(rec, req) + return rec +} + +func relayErr(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + var e struct { + Error string `json:"error"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &e) + return e.Error +} + +// TestHubRelayedWebhookAcceptedWithoutSecret: a hosted spoke with no +// GITHUB_WEBHOOK_SECRET applies a delivery its hub relayed and signed with +// this hive's relay key (#17911). +func TestHubRelayedWebhookAcceptedWithoutSecret(t *testing.T) { + const key = "relay-key-h1" + withWebhookRelayKey(t, key) + s := newGitHubWebhookTestServer(t) + + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + ts := strconv.FormatInt(time.Now().Unix(), 10) + rec := postRelayedWebhook(s, "pull_request", "relay-accept-1", ts, signWebhookRelayForTest(key, ts, "relay-accept-1", body), body) + if rec.Code != http.StatusOK { + t.Fatalf("relay-signed delivery: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + var got struct { + Status string `json:"status"` + Invalidated []int `json:"invalidated"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Status != "ok" || !reflect.DeepEqual(got.Invalidated, []int{5}) { + t.Fatalf("relay-signed delivery = %+v, want ok/[5]", got) + } +} + +// TestHubRelayedWebhookRejections: tampered bodies, wrong keys, stale +// timestamps, missing or replayed delivery IDs are each rejected with their +// own reason. +func TestHubRelayedWebhookRejections(t *testing.T) { + const key = "relay-key-h1" + withWebhookRelayKey(t, key) + s := newGitHubWebhookTestServer(t) + + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + now := strconv.FormatInt(time.Now().Unix(), 10) + + t.Run("tampered body", func(t *testing.T) { + tampered := []byte(strings.Replace(string(body), `"number":5`, `"number":6`, 1)) + rec := postRelayedWebhook(s, "pull_request", "tamper-1", now, signWebhookRelayForTest(key, now, "tamper-1", body), tampered) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonBadSignature) { + t.Fatalf("tampered body: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonBadSignature) + } + }) + t.Run("another hive's key", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "other-1", now, signWebhookRelayForTest("relay-key-h2", now, "other-1", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonBadSignature) { + t.Fatalf("other hive's key: %d %s, want 401", rec.Code, rec.Body.String()) + } + }) + t.Run("delivery ID swapped after signing", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "swapped-2", now, signWebhookRelayForTest(key, now, "swapped-1", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonBadSignature) { + t.Fatalf("swapped delivery: %d %s, want 401", rec.Code, rec.Body.String()) + } + }) + t.Run("missing relay signature", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "unsigned-1", now, "", body) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("unsigned relay: status = %d, want 401", rec.Code) + } + }) + t.Run("stale timestamp", func(t *testing.T) { + old := strconv.FormatInt(time.Now().Add(-10*time.Minute).Unix(), 10) + rec := postRelayedWebhook(s, "pull_request", "stale-1", old, signWebhookRelayForTest(key, old, "stale-1", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonStale) { + t.Fatalf("stale timestamp: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonStale) + } + future := strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10) + rec = postRelayedWebhook(s, "pull_request", "stale-2", future, signWebhookRelayForTest(key, future, "stale-2", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonStale) { + t.Fatalf("future timestamp: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonStale) + } + }) + t.Run("missing delivery ID", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "", now, signWebhookRelayForTest(key, now, "", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonNoDelivery) { + t.Fatalf("missing delivery: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonNoDelivery) + } + }) + t.Run("replayed delivery ID", func(t *testing.T) { + sig := signWebhookRelayForTest(key, now, "replay-1", body) + if rec := postRelayedWebhook(s, "pull_request", "replay-1", now, sig, body); rec.Code != http.StatusOK { + t.Fatalf("first delivery: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + rec := postRelayedWebhook(s, "pull_request", "replay-1", now, sig, body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonReplayed) { + t.Fatalf("replay: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonReplayed) + } + }) +} + +// TestHubRelayedWebhookFailuresAreCounted: a rejected relay counts as a +// failed delivery, the same signal a bad GitHub signature feeds the cadence +// mode, so a burst of forged relays degrades to polling and is visible. +func TestHubRelayedWebhookFailuresAreCounted(t *testing.T) { + const key = "relay-key-h1" + withWebhookRelayKey(t, key) + t.Cleanup(github.ResetWebhookTrackerForTest()) + s := newGitHubWebhookTestServer(t) + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + now := strconv.FormatInt(time.Now().Unix(), 10) + + if rec := postRelayedWebhook(s, "pull_request", "count-1", now, signWebhookRelayForTest(key, now, "count-1", body), body); rec.Code != http.StatusOK { + t.Fatalf("good relay: status = %d (%s)", rec.Code, rec.Body.String()) + } + if _, _, recovered := github.ObserveWebhookMode(); !recovered { + t.Fatal("a verified relay should engage webhook mode") + } + if rec := postRelayedWebhook(s, "pull_request", "count-2", now, signWebhookRelayForTest("wrong", now, "count-2", body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("forged relay: status = %d, want 401", rec.Code) + } + if _, degraded, _ := github.ObserveWebhookMode(); !degraded { + t.Fatal("a rejected relay after the last good one must be counted as a failed delivery") + } +} + +// TestGitHubSignedWebhookUnchangedWithRelayKey: a spoke with +// GITHUB_WEBHOOK_SECRET keeps accepting GitHub-signed deliveries exactly as +// before, relay key or not — no relay headers needed and no replay check. +func TestGitHubSignedWebhookUnchangedWithRelayKey(t *testing.T) { + withWebhookRelayKey(t, "relay-key-h1") + const secret = "s3cr3t" + t.Setenv(githubWebhookSecretEnvVar, secret) + s := newGitHubWebhookTestServer(t) + + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + for i := 0; i < 2; i++ { + req := httptest.NewRequest(http.MethodPost, githubWebhookPath, bytes.NewReader(body)) + req.Header.Set("X-GitHub-Event", "pull_request") + req.Header.Set("X-GitHub-Delivery", "gh-direct-1") + req.Header.Set("X-Hub-Signature-256", signGitHubWebhookForTest(secret, body)) + rec := httptest.NewRecorder() + s.handleGitHubWebhook(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GitHub-signed delivery %d: status = %d, want 200 (%s)", i+1, rec.Code, rec.Body.String()) + } + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("wrong", body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("bad GitHub signature: status = %d, want 401", rec.Code) + } +} + +// TestWebhooksConfiguredByRelayKey: the relay key alone turns webhooks on +// for event-driven review dispatch and /api/status.webhooks.verification. +func TestWebhooksConfiguredByRelayKey(t *testing.T) { + withWebhookRelayKey(t, "") + if githubWebhooksConfigured() { + t.Fatal("no secret and no relay key must leave webhooks off") + } + s := newGitHubWebhookTestServer(t) + body := []byte(`{}`) + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("x", body), body); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no secret, no relay key: status = %d, want 503", rec.Code) + } + + if _, err := github.AdoptWebhookRelayKey("relay-key-h1"); err != nil { + t.Fatal(err) + } + if !githubWebhooksConfigured() { + t.Fatal("a hub relay key must turn webhooks on") + } + + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "relay-test"} + s.statusMu.Unlock() + rec := httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status?fields=webhooks", nil)) + var status struct { + Webhooks struct { + Verification string `json:"verification"` + } `json:"webhooks"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status.Webhooks.Verification != github.WebhookVerificationRelay { + t.Fatalf("webhooks.verification = %q, want %q", status.Webhooks.Verification, github.WebhookVerificationRelay) + } +} + +func TestWebhookRelayReplayCacheIsBounded(t *testing.T) { + var c webhookRelayReplayCache + now := time.Now() + for i := 0; i < webhookRelayReplayMax+10; i++ { + if !c.firstSeen(strconv.Itoa(i), now.Add(time.Duration(i)*time.Millisecond)) { + t.Fatalf("id %d reported as seen", i) + } + } + if len(c.seen) > webhookRelayReplayMax { + t.Fatalf("cache holds %d ids, want at most %d", len(c.seen), webhookRelayReplayMax) + } + later := now.Add(webhookRelayReplayTTL + time.Hour) + if !c.firstSeen(strconv.Itoa(webhookRelayReplayMax+5), later) { + t.Fatal("an id older than the replay TTL must be accepted again") + } +} diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 155572129d..aac6014c49 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -45,6 +45,10 @@ type WebhookHealth struct { // WebhookModeStatus is the reconcile cadence the health drives: // `mode` webhook|polling and `since` (hivecommons/hive#13381). WebhookModeStatus + // Verification is how this spoke authenticates deliveries: + // "github-signed", "relay (hub-signed)" or "none" (WebhookVerification, + // hivecommons/hive#17911). + Verification string `json:"verification,omitempty"` } // WebhookEventResult reports what one delivered GitHub webhook did. @@ -402,7 +406,7 @@ func (t *webhookTracker) snapshot() WebhookHealth { out := WebhookHealth{Healthy: healthy, Events1h: len(t.events), Invalidations1h: len(t.invalidations), TargetedRefreshes1h: len(t.refreshes), TrackerInvalidations1h: len(t.trackerDrops), CIEvents1h: len(t.ciEvents), CIEventsApplied1h: len(t.ciApplied), CIRefetches1h: len(t.ciRefetches), - WebhookModeStatus: t.mode.status(t.window, now)} + WebhookModeStatus: t.mode.status(t.window, now), Verification: WebhookVerification()} if !latest.IsZero() { at := latest.UTC() out.LastEventAt = &at diff --git a/src/pkg/github/webhook_relay_key.go b/src/pkg/github/webhook_relay_key.go new file mode 100644 index 0000000000..d850abdb66 --- /dev/null +++ b/src/pkg/github/webhook_relay_key.go @@ -0,0 +1,134 @@ +package github + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "sync" +) + +// Hub webhook relay key (hivecommons/hive#17911). +// +// A hosted spoke never holds the GitHub App webhook secret, so it cannot +// verify X-Hub-Signature-256 on a delivery the hub relays. The hub instead +// signs each relay with a key of this hive's own, delivered over the +// heartbeat (HeartbeatGitHubAppConfig.WebhookRelayKey); the receiver +// (pkg/dashboard api_github_webhook.go) verifies that signature. The key is +// persisted 0600 on the data volume so a restarted spoke accepts relays +// before its first heartbeat, and is only ever reported by fingerprint. + +const defaultWebhookRelayKeyPath = "/data/webhook-relay.key" + +// WebhookSecretEnvVar names the GitHub App webhook HMAC secret a self-hosted +// spoke verifies /api/webhook/github deliveries with. +const WebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + +// Values of /api/status.webhooks.verification: how this spoke authenticates +// GitHub webhook deliveries. +const ( + // WebhookVerificationGitHub: GITHUB_WEBHOOK_SECRET is set and GitHub's + // X-Hub-Signature-256 is verified (self-hosted, or a hub relay to a spoke + // that holds the secret). + WebhookVerificationGitHub = "github-signed" + // WebhookVerificationRelay: no secret, but the hub delivered this hive's + // relay key, so hub-relayed deliveries are verified by the hub's signature. + WebhookVerificationRelay = "relay (hub-signed)" + // WebhookVerificationNone: neither; every delivery is rejected with 503. + WebhookVerificationNone = "none" +) + +var webhookRelayKeyState = struct { + mu sync.Mutex + path string + key string + loaded bool +}{path: defaultWebhookRelayKeyPath} + +func webhookRelayKeyLocked() string { + st := &webhookRelayKeyState + if !st.loaded { + st.loaded = true + if data, err := os.ReadFile(st.path); err == nil { + st.key = strings.TrimSpace(string(data)) + } + } + return st.key +} + +// WebhookRelayKey is the hub relay key this spoke holds, or "". +func WebhookRelayKey() string { + webhookRelayKeyState.mu.Lock() + defer webhookRelayKeyState.mu.Unlock() + return webhookRelayKeyLocked() +} + +// AdoptWebhookRelayKey stores a relay key delivered by the hub and reports +// whether it replaced the one held (a first delivery or a rotation). +// +// An empty key means "the hub is not speaking to this field", never "clear +// it", so a beat without the key can never blank a working one. A key that +// cannot be persisted is still adopted in memory (relays verify until the +// next restart) and the write error is returned for the caller to log. +func AdoptWebhookRelayKey(key string) (bool, error) { + key = strings.TrimSpace(key) + if key == "" { + return false, nil + } + st := &webhookRelayKeyState + st.mu.Lock() + defer st.mu.Unlock() + if webhookRelayKeyLocked() == key { + return false, nil + } + st.key = key + if err := os.MkdirAll(filepath.Dir(st.path), 0o700); err != nil { + return true, err + } + tmp := st.path + ".tmp" + if err := os.WriteFile(tmp, []byte(key+"\n"), 0o600); err != nil { + return true, err + } + return true, os.Rename(tmp, st.path) +} + +// WebhookRelayKeyFingerprint is the NON-SECRET "sha256:<hex>" of the relay key +// this spoke holds, reported on every heartbeat so the hub delivers the key +// only when the spoke lacks it or holds a rotated-out one. Empty when none. +func WebhookRelayKeyFingerprint() string { + key := WebhookRelayKey() + if key == "" { + return "" + } + sum := sha256.Sum256([]byte(key)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// WebhookVerification reports how this spoke verifies webhook deliveries +// (WebhookVerification* above). Only the mode ever leaves this function, never +// the secret or the key. +func WebhookVerification() string { + if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + return WebhookVerificationGitHub + } + if WebhookRelayKey() != "" { + return WebhookVerificationRelay + } + return WebhookVerificationNone +} + +// SetWebhookRelayKeyPathForTest points the relay key store at path, forgets +// any key held in memory, and returns a func that restores the previous state. +func SetWebhookRelayKeyPathForTest(path string) func() { + st := &webhookRelayKeyState + st.mu.Lock() + prevPath, prevKey, prevLoaded := st.path, st.key, st.loaded + st.path, st.key, st.loaded = path, "", false + st.mu.Unlock() + return func() { + st.mu.Lock() + st.path, st.key, st.loaded = prevPath, prevKey, prevLoaded + st.mu.Unlock() + } +} diff --git a/src/pkg/github/webhook_relay_key_test.go b/src/pkg/github/webhook_relay_key_test.go new file mode 100644 index 0000000000..4775181aab --- /dev/null +++ b/src/pkg/github/webhook_relay_key_test.go @@ -0,0 +1,70 @@ +package github + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" +) + +// TestAdoptWebhookRelayKey: the hub relay key delivered over the heartbeat is +// persisted 0600 and survives a restart, a rotation replaces it, and a beat +// that carries no key never blanks a working one (#17911). +func TestAdoptWebhookRelayKey(t *testing.T) { + path := filepath.Join(t.TempDir(), "webhook-relay.key") + t.Cleanup(SetWebhookRelayKeyPathForTest(path)) + t.Setenv(WebhookSecretEnvVar, "") + + if WebhookRelayKey() != "" || WebhookRelayKeyFingerprint() != "" || WebhookVerification() != WebhookVerificationNone { + t.Fatal("fresh spoke must hold no relay key") + } + + changed, err := AdoptWebhookRelayKey("key-1") + if err != nil || !changed { + t.Fatalf("first delivery: changed=%v err=%v, want adopted", changed, err) + } + if WebhookRelayKey() != "key-1" || WebhookVerification() != WebhookVerificationRelay { + t.Fatalf("after delivery: key=%q verification=%q", WebhookRelayKey(), WebhookVerification()) + } + fp := WebhookRelayKeyFingerprint() + // Same format the hub compares against (pkg/hub webhookRelayKeyFingerprint). + sum := sha256.Sum256([]byte("key-1")) + if want := "sha256:" + hex.EncodeToString(sum[:]); fp != want { + t.Fatalf("fingerprint = %q, want %q", fp, want) + } + if fi, err := os.Stat(path); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("relay key file: %v, want mode 0600", err) + } + + if changed, err := AdoptWebhookRelayKey("key-1"); err != nil || changed { + t.Fatalf("redelivery of the same key: changed=%v err=%v, want no change", changed, err) + } + for _, empty := range []string{"", " "} { + if changed, err := AdoptWebhookRelayKey(empty); err != nil || changed { + t.Fatalf("empty delivery %q: changed=%v err=%v", empty, changed, err) + } + if WebhookRelayKey() != "key-1" { + t.Fatal("an empty delivery blanked a working relay key") + } + } + + if changed, err := AdoptWebhookRelayKey("key-2"); err != nil || !changed { + t.Fatalf("rotation: changed=%v err=%v, want adopted", changed, err) + } + if WebhookRelayKeyFingerprint() == fp { + t.Fatal("fingerprint did not change after rotation") + } + + // A restarted spoke reads the persisted key before its first heartbeat. + restore := SetWebhookRelayKeyPathForTest(path) + defer restore() + if WebhookRelayKey() != "key-2" { + t.Fatalf("after restart key = %q, want the rotated key from disk", WebhookRelayKey()) + } + + t.Setenv(WebhookSecretEnvVar, "s3cr3t") + if WebhookVerification() != WebhookVerificationGitHub { + t.Fatalf("with GITHUB_WEBHOOK_SECRET set verification = %q, want %q", WebhookVerification(), WebhookVerificationGitHub) + } +} diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index 242a9a3383..110e36b44c 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -343,6 +343,7 @@ type HeartbeatPayload struct { GatewayHealth []inferencehealth.GatewayStatus `json:"gateway_health,omitempty"` GitHubAppKeyFingerprint string `json:"github_app_key_fingerprint,omitempty"` GitHubAppKeyPerHive bool `json:"github_app_key_per_hive,omitempty"` + WebhookRelayKeyFingerprint string `json:"webhook_relay_key_fingerprint,omitempty"` GitHubAppID int64 `json:"github_app_id,omitempty"` GitHubAppSlug string `json:"github_app_slug,omitempty"` GitHubInstallationID int64 `json:"github_installation_id,omitempty"` @@ -411,6 +412,10 @@ type HeartbeatGitHubAppConfig struct { BaseURL string `json:"base_url,omitempty"` AdditionalKeys []HeartbeatAppKey `json:"additional_keys,omitempty"` SecondaryKey *HeartbeatAppKey `json:"secondary_key,omitempty"` + // WebhookRelayKey is this hive's own key for verifying webhooks the hub + // relays to it (webhook_relay_key.go). Never the App webhook secret. + // Empty means "leave the spoke's key unchanged". + WebhookRelayKey string `json:"webhook_relay_key,omitempty"` } type HeartbeatAppKey struct { diff --git a/src/pkg/hub/lite_enrollment.go b/src/pkg/hub/lite_enrollment.go index 9e66e2207d..a1b5af3544 100644 --- a/src/pkg/hub/lite_enrollment.go +++ b/src/pkg/hub/lite_enrollment.go @@ -330,6 +330,7 @@ func (s *HubServer) requestHostedLiteSpoke(username, owner, repo, host string, i ClusterID: clusterID, IsPublic: false, AutoUpgrade: true, + WebhookRelayKey: newWebhookRelayKey(), } if host != "" && host != publicGitHubHost { h.GitHubHost = host diff --git a/src/pkg/hub/pool_replenisher.go b/src/pkg/hub/pool_replenisher.go index 19668a5ae8..7dd44ee912 100644 --- a/src/pkg/hub/pool_replenisher.go +++ b/src/pkg/hub/pool_replenisher.go @@ -145,9 +145,10 @@ func (s *HubServer) seedPlaceholder(cluster *ClusterConfig) error { // Pool slots track stable and auto-upgrade so inventory never rots on // an old image while waiting to be assigned — same posture as the // hand-seeded fleet. - TrackedChannel: "stable", - AutoUpgrade: true, - IsPublic: false, + TrackedChannel: "stable", + AutoUpgrade: true, + IsPublic: false, + WebhookRelayKey: newWebhookRelayKey(), } if err := saveSaaSHive(h); err != nil { return fmt.Errorf("save placeholder record: %w", err) diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index 71e215330d..a6713566dc 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -1552,8 +1552,9 @@ func (s *HubServer) handleCreateHive(w http.ResponseWriter, r *http.Request) { IsPublic: req.IsPublic == nil || *req.IsPublic, // Per-hive GitHub host override (empty = cluster default). Lets a // public-GitHub org provision correctly on a GHE-defaulted cluster. - GitHubBaseURL: req.GitHubBaseURL, - GitHubAPIURL: req.GitHubAPIURL, + GitHubBaseURL: req.GitHubBaseURL, + GitHubAPIURL: req.GitHubAPIURL, + WebhookRelayKey: newWebhookRelayKey(), } if err := saveSaaSHive(h); err != nil { @@ -1627,8 +1628,12 @@ func (s *HubServer) handleHiveStatus(w http.ResponseWriter, r *http.Request) { http.Error(w, `{"error":"access denied"}`, http.StatusForbidden) return } + // The relay key authenticates webhooks to this hive's spoke; it is + // delivered only over the spoke's own heartbeat, never to a browser. + out := *h + out.WebhookRelayKey = "" w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(h) + _ = json.NewEncoder(w).Encode(&out) } // handleOpenHive is the SSO handoff entry point: a hub-authenticated user hits diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index b5de4a99ca..75939a069a 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -797,6 +797,15 @@ type SaaSHive struct { // delivery (see AppIdentityDelivered), so a lost beat is retried. PendingAppConfig *PendingAppIdentity `json:"pending_app_config,omitempty"` + // WebhookRelayKey is this hive's own HMAC key for webhooks the hub relays + // to its spoke (webhook_relay_key.go, hivecommons/hive#17911). Minted at + // provisioning (or on the first heartbeat of a hive provisioned before + // relay keys existed) and delivered over the heartbeat. Per hive so one + // spoke's key can never sign a relay another spoke accepts, and so no + // spoke ever needs the GitHub App webhook secret. Replacing or clearing it + // rotates the key: the next beat delivers the new one. + WebhookRelayKey string `json:"webhook_relay_key,omitempty"` + // RequestedRestartAt (RFC3339) arms a rolling restart of this hive's spoke, // delivered on every heartbeat inside spokeRestartDeliveryWindow and then // cleared. Durable on the hive record for the same reason as @@ -2081,15 +2090,26 @@ func saveSaaSHive(h *SaaSHive) error { return fmt.Errorf("invalid hive ID for save: %q", h.ID) } dir := filepath.Join(saasHivesDir, h.ID) - // Best-effort: a failed mkdir surfaces via the WriteFile error below. - _ = os.MkdirAll(dir, 0o755) + // meta.json carries WebhookRelayKey, so it is written rw------- under an + // rwx------ directory like the other hub secret stores (cluster App keys, + // generation stores). Best-effort: a failed mkdir surfaces via the + // WriteFile error below. + _ = os.MkdirAll(dir, 0o700) + // MkdirAll leaves pre-existing directories untouched, so tighten hives + // provisioned before meta.json carried key material. Only group/other + // bits are stripped: owner bits are left as found so a directory an + // operator (or a test) made read-only is never silently widened back to + // writable, and the WriteFile below still reports the failure. + if fi, err := os.Stat(dir); err == nil && fi.Mode().Perm()&0o077 != 0 { + _ = os.Chmod(dir, fi.Mode().Perm()&^0o077) + } data, err := json.MarshalIndent(h, "", " ") if err != nil { return err } path := filepath.Join(dir, "meta.json") tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, data, 0o644); err != nil { + if err := os.WriteFile(tmpPath, data, 0o600); err != nil { return err } if err := os.Rename(tmpPath, path); err != nil { diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index aedd7a7e1f..5638807292 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -3359,6 +3359,21 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { resp.GitHubAppConfig.SecondaryKey = secCfg } + // Per-hive webhook relay key (#17911), attached the same independent way as + // the secondary key: it rides whatever config the branches above produced, + // or an otherwise-empty one, and only until the spoke reports holding it. + if relayKey := s.webhookRelayKeyForHeartbeat(&payload); relayKey != "" { + if resp.GitHubAppConfig == nil { + resp.GitHubAppConfig = &HeartbeatGitHubAppConfig{} + } + resp.GitHubAppConfig.WebhookRelayKey = relayKey + s.logger.Info("heartbeat: delivering webhook relay key to spoke", + "hive_id", payload.HiveID, + "fingerprint", webhookRelayKeyFingerprint(relayKey), + "spoke_fingerprint", payload.WebhookRelayKeyFingerprint, + ) + } + // SECURITY (C1/N3, CWE-200/639): the fleet-wide additional-key broadcast that // once ran here was removed. It attached every OTHER tenant's App private key // to every heartbeat, and — because the handler trusts the body-supplied diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 86c71807b5..2ca00c37cd 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1056,6 +1056,13 @@ type HeartbeatPayload struct { // its cluster. The hub honours it as an override: a deliberate per-hive // credential is never overwritten by the cluster default. GitHubAppKeyPerHive bool `json:"github_app_key_per_hive,omitempty"` + // WebhookRelayKeyFingerprint is the NON-SECRET "sha256:<hex>" of the hub + // webhook relay key this spoke holds (github.WebhookRelayKeyFingerprint). + // The hub delivers the key only while this does not match its record, so + // a missing or rotated-out key is repaired on the next beat. Empty means + // no key (or a spoke too old to report one); the key itself never travels + // spoke → hub. + WebhookRelayKeyFingerprint string `json:"webhook_relay_key_fingerprint,omitempty"` // GitHubAppID is the App ID this spoke believes it is authenticating AS — // cfg.GitHub.AppID, non-secret. It exists to make one specific distinction // decidable on the hub: a per-hive key that is WRONG versus a per-hive key @@ -2752,6 +2759,15 @@ type HeartbeatGitHubAppConfig struct { // nil for every hive without a SecondaryAppID, which is the entire fleet // today, so the heartbeat payload is byte-identical for them. SecondaryKey *HeartbeatAppKey `json:"secondary_key,omitempty"` + // WebhookRelayKey is this hive's OWN key for verifying the signature the + // hub adds to every GitHub webhook it relays here (hivecommons/hive#17911). + // It is per hive and is never the GitHub App webhook secret, so it cannot + // sign anything another spoke would accept. Secret — never logged; the + // spoke persists it 0600 (github.AdoptWebhookRelayKey). + // + // Empty means "leave the spoke's key unchanged" — never a way to blank a + // working key. + WebhookRelayKey string `json:"webhook_relay_key,omitempty"` } // HeartbeatAppKey is one (app_id, private key) pair the hub delivers alongside diff --git a/src/pkg/hub/webhook.go b/src/pkg/hub/webhook.go index b6dbe65dbd..d177a66280 100644 --- a/src/pkg/hub/webhook.go +++ b/src/pkg/hub/webhook.go @@ -129,8 +129,11 @@ var spokeWebhookRelayClient = &http.Client{ // relayWebhookToSpoke forwards a verified PR-shaped delivery, byte for byte // and with its original signature, to the hive that manages the repository so -// the spoke can invalidate its cached PR state (hivecommons/hive#11177). The -// spoke re-verifies X-Hub-Signature-256 with the same App webhook secret. +// the spoke can invalidate its cached PR state (hivecommons/hive#11177). A +// self-hosted spoke re-verifies X-Hub-Signature-256 with its own copy of the +// App webhook secret; a hosted spoke never holds that secret, so the hub also +// signs each relay with the target hive's own relay key +// (signWebhookRelayHeaders) and the spoke verifies that instead. // Best effort and asynchronous: GitHub's delivery must not wait on a spoke. func (s *HubServer) relayWebhookToSpoke(event string, header http.Header, body []byte) { var evt struct { @@ -184,8 +187,13 @@ func (s *HubServer) relayWebhookToSpoke(event string, header http.Header, body [ for _, t := range targets { hiveID, dashboardURL := t.hiveID, t.dashboardURL spokeURL := strings.TrimRight(dashboardURL, "/") + spokeWebhookPath - s.logger.Info("relaying webhook to spoke", "hive_id", hiveID, "spoke_url", spokeURL, "event", event, "delivery", delivery) + relayKey := webhookRelayKeyForHive(hiveID) + s.logger.Info("relaying webhook to spoke", "hive_id", hiveID, "spoke_url", spokeURL, "event", event, "delivery", delivery, "relay_signed", relayKey != "") payload := append([]byte(nil), body...) + hdr := forward.Clone() + if relayKey != "" { + signWebhookRelayHeaders(hdr, relayKey, time.Now(), delivery, payload) + } go func() { ctx, cancel := context.WithTimeout(context.Background(), webhookPushTimeout) defer cancel() @@ -198,7 +206,7 @@ func (s *HubServer) relayWebhookToSpoke(event string, header http.Header, body [ s.logger.Warn("failed to build spoke webhook relay request", "hive_id", hiveID, "error", err) return } - req.Header = forward.Clone() + req.Header = hdr resp, err := spokeWebhookRelayClient.Do(req) if err != nil { s.logger.Warn("spoke webhook relay failed", "hive_id", hiveID, "event", event, "error", err) diff --git a/src/pkg/hub/webhook_relay_key.go b/src/pkg/hub/webhook_relay_key.go new file mode 100644 index 0000000000..e0c9b9479c --- /dev/null +++ b/src/pkg/hub/webhook_relay_key.go @@ -0,0 +1,121 @@ +package hub + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "net/http" + "strconv" + "time" +) + +// Hub-signed webhook relay (hivecommons/hive#17911). +// +// A hosted spoke must be able to verify a delivery the hub relays to it, but +// it must never hold the GitHub App webhook secret: that one secret +// authenticates GitHub deliveries for the whole App, so any spoke holding it +// could forge deliveries to every other spoke's public /api/webhook/github. +// Instead every hosted hive gets its OWN random relay key, stored on its hub +// record (SaaSHive.WebhookRelayKey) and delivered over its per-hive +// authenticated heartbeat. The hub signs each relay with the target hive's +// key; a key held by one spoke cannot sign anything another spoke accepts. +// +// Signature scheme (mirrored by pkg/dashboard verifyWebhookRelaySignature): +// +// X-Hive-Relay-Timestamp: <unix seconds at relay time> +// X-Hive-Relay-Signature-256: sha256=<hex HMAC-SHA256(key, timestamp + "." + X-GitHub-Delivery + "." + raw body)> +// +// The timestamp and delivery ID are inside the MAC so the spoke can reject +// stale relays and replayed delivery IDs without trusting unsigned headers. +// GitHub's own headers (X-Hub-Signature-256 included) are forwarded +// unchanged next to these, so a self-hosted spoke is unaffected. +const ( + webhookRelaySignatureHeader = "X-Hive-Relay-Signature-256" + webhookRelayTimestampHeader = "X-Hive-Relay-Timestamp" +) + +// newWebhookRelayKey mints a 256-bit relay key, hex encoded. +func newWebhookRelayKey() string { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "" + } + return hex.EncodeToString(b) +} + +// webhookRelayKeyFingerprint is the NON-SECRET identifier of a relay key the +// spoke reports back on every heartbeat (webhook_relay_key_fingerprint), so the +// hub delivers the key only when the spoke does not already hold it. Same +// format as pkg/github WebhookRelayKeyFingerprint. +func webhookRelayKeyFingerprint(key string) string { + if key == "" { + return "" + } + sum := sha256.Sum256([]byte(key)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// webhookRelaySignature computes the relay MAC described above. +func webhookRelaySignature(key, timestamp, delivery string, body []byte) string { + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(timestamp)) + mac.Write([]byte(".")) + mac.Write([]byte(delivery)) + mac.Write([]byte(".")) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +// signWebhookRelayHeaders adds the relay timestamp and signature for one +// target hive's key to an outgoing relay. +func signWebhookRelayHeaders(h http.Header, key string, now time.Time, delivery string, body []byte) { + ts := strconv.FormatInt(now.Unix(), 10) + h.Set(webhookRelayTimestampHeader, ts) + h.Set(webhookRelaySignatureHeader, webhookRelaySignature(key, ts, delivery, body)) +} + +// webhookRelayKeyForHive is the relay key on a hive's hub record, or "" for a +// hive the hub did not provision (a self-hosted spoke keeps verifying GitHub's +// signature with its own secret). +func webhookRelayKeyForHive(hiveID string) string { + if h := loadSaaSHive(hiveID); h != nil { + return h.WebhookRelayKey + } + return "" +} + +// webhookRelayKeyForHeartbeat returns the relay key to deliver on this beat, +// or "" when the spoke already reports holding it. +// +// A hosted hive provisioned before relay keys existed has none on its record; +// one is minted here on its first beat so the fix reaches the existing fleet. +// Rotation is the same path: replace (or clear) SaaSHive.WebhookRelayKey and +// the spoke's fingerprint stops matching, so the next beat delivers the new +// key. The beat is authenticated with the per-hive bearer before this runs +// (verifyHeartbeatBearer), so a spoke can only ever be handed its own key. +func (s *HubServer) webhookRelayKeyForHeartbeat(p *HeartbeatPayload) string { + if p == nil || p.HiveID == "" { + return "" + } + h := loadSaaSHive(p.HiveID) + if h == nil { + return "" + } + if h.WebhookRelayKey == "" { + key := newWebhookRelayKey() + if key == "" { + return "" + } + h.WebhookRelayKey = key + if err := saveSaaSHive(h); err != nil { + s.logger.Warn("webhook relay: could not store a new relay key on the hive record", "hive_id", p.HiveID, "error", err) + return "" + } + s.logger.Info("webhook relay: minted relay key for hive", "hive_id", p.HiveID, "fingerprint", webhookRelayKeyFingerprint(key)) + } + if p.WebhookRelayKeyFingerprint == webhookRelayKeyFingerprint(h.WebhookRelayKey) { + return "" + } + return h.WebhookRelayKey +} diff --git a/src/pkg/hub/webhook_relay_key_test.go b/src/pkg/hub/webhook_relay_key_test.go new file mode 100644 index 0000000000..4e07fe1859 --- /dev/null +++ b/src/pkg/hub/webhook_relay_key_test.go @@ -0,0 +1,286 @@ +package hub + +import ( + "bytes" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +type relayedSignedWebhook struct { + header http.Header + body []byte +} + +func captureRelaySpoke(t *testing.T) (*httptest.Server, chan relayedSignedWebhook) { + t.Helper() + got := make(chan relayedSignedWebhook, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + got <- relayedSignedWebhook{header: r.Header.Clone(), body: body} + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + return srv, got +} + +func useTempSaaSHivesDir(t *testing.T) { + t.Helper() + old := saasHivesDir + saasHivesDir = t.TempDir() + t.Cleanup(func() { saasHivesDir = old }) +} + +// TestRelayWebhookSignsWithTargetHiveRelayKey: the relay to a hosted hive +// carries a hub signature that verifies with THAT hive's relay key and not +// with another hive's, binds the delivery ID and a fresh timestamp, and still +// forwards GitHub's own signature unchanged (#17911). +func TestRelayWebhookSignsWithTargetHiveRelayKey(t *testing.T) { + const secret = "s3cr3t" + t.Setenv(webhookSecretEnvVar, secret) + allowLoopbackRelay(t) + useTempSaaSHivesDir(t) + for id, key := range map[string]string{"h1": "relay-key-h1", "h2": "relay-key-h2"} { + if err := saveSaaSHive(&SaaSHive{ID: id, WebhookRelayKey: key}); err != nil { + t.Fatal(err) + } + } + spokeSrv, got := captureRelaySpoke(t) + + s := newWebhookHub() + s.registry.Hives = []RegistryEntry{{ID: "h1", Org: "acme", Repos: []string{"widgets"}, DashboardURL: spokeSrv.URL}} + + body := []byte(`{"action":"synchronize","number":7,"repository":{"name":"widgets","full_name":"acme/widgets","owner":{"login":"acme"}}}`) + ghSig := signWebhook(secret, body) + req := httptest.NewRequest(http.MethodPost, "/api/github/webhook", bytes.NewReader(body)) + req.Header.Set("X-GitHub-Event", "pull_request") + req.Header.Set("X-GitHub-Delivery", "delivery-1") + req.Header.Set("X-Hub-Signature-256", ghSig) + before := time.Now().Unix() + rec := httptest.NewRecorder() + s.handleGitHubWebhook(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + + var r relayedSignedWebhook + select { + case r = <-got: + case <-time.After(5 * time.Second): + t.Fatal("delivery was not relayed to the managing spoke") + } + if r.header.Get("X-Hub-Signature-256") != ghSig || !bytes.Equal(r.body, body) { + t.Fatal("GitHub's signature and body must still be forwarded unchanged") + } + ts := r.header.Get(webhookRelayTimestampHeader) + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil || sec < before || sec > time.Now().Unix() { + t.Fatalf("relay timestamp = %q, want unix seconds at relay time", ts) + } + sig := r.header.Get(webhookRelaySignatureHeader) + if sig == "" || sig != webhookRelaySignature("relay-key-h1", ts, "delivery-1", body) { + t.Fatalf("relay signature %q does not verify with the target hive's relay key", sig) + } + if sig == webhookRelaySignature("relay-key-h2", ts, "delivery-1", body) { + t.Fatal("relay signature verifies with another hive's relay key") + } + if sig == webhookRelaySignature("relay-key-h1", ts, "delivery-2", body) { + t.Fatal("relay signature must bind the delivery ID") + } +} + +// TestRelayWebhookUnsignedForHiveWithoutRelayKey: a hive the hub did not +// provision (no record, so no relay key) gets the relay exactly as before. +func TestRelayWebhookUnsignedForHiveWithoutRelayKey(t *testing.T) { + allowLoopbackRelay(t) + useTempSaaSHivesDir(t) + spokeSrv, got := captureRelaySpoke(t) + + s := newWebhookHub() + s.registry.Hives = []RegistryEntry{{ID: "selfhosted", Org: "acme", Repos: []string{"widgets"}, DashboardURL: spokeSrv.URL}} + hdr := http.Header{} + hdr.Set("X-GitHub-Delivery", "d") + s.relayWebhookToSpoke("push", hdr, []byte(`{"repository":{"name":"widgets","full_name":"acme/widgets","owner":{"login":"acme"}}}`)) + select { + case r := <-got: + if r.header.Get(webhookRelaySignatureHeader) != "" || r.header.Get(webhookRelayTimestampHeader) != "" { + t.Fatalf("hive without a relay key got relay headers: %v", r.header) + } + case <-time.After(5 * time.Second): + t.Fatal("delivery was not relayed") + } +} + +// TestWebhookRelayKeyForHeartbeat: the relay key is delivered until the spoke +// reports holding it, minted for a hosted hive provisioned before relay keys +// existed, re-delivered after a rotation, and never offered to a hive the hub +// has no record for. +func TestWebhookRelayKeyForHeartbeat(t *testing.T) { + useTempSaaSHivesDir(t) + s := &HubServer{logger: slog.Default()} + + if got := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "unknown"}); got != "" { + t.Fatalf("hive without a record got a relay key") + } + + if err := saveSaaSHive(&SaaSHive{ID: "legacy"}); err != nil { + t.Fatal(err) + } + minted := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy"}) + if len(minted) != 64 { + t.Fatalf("legacy hive: delivered %q, want a minted 256-bit hex key", minted) + } + if h := loadSaaSHive("legacy"); h == nil || h.WebhookRelayKey != minted { + t.Fatal("minted relay key was not stored on the hive record") + } + if again := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy"}); again != minted { + t.Fatal("a spoke that has not confirmed the key must get the same key again, not a new one") + } + if got := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy", WebhookRelayKeyFingerprint: webhookRelayKeyFingerprint(minted)}); got != "" { + t.Fatal("key re-delivered to a spoke that already holds it") + } + + // Rotation: a new key on the record no longer matches the spoke's + // fingerprint, so the next beat delivers it. + h := loadSaaSHive("legacy") + h.WebhookRelayKey = "rotated-key" + if err := saveSaaSHive(h); err != nil { + t.Fatal(err) + } + if got := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy", WebhookRelayKeyFingerprint: webhookRelayKeyFingerprint(minted)}); got != "rotated-key" { + t.Fatalf("after rotation delivered %q, want the rotated key", got) + } +} + +// TestProvisionedHiveRecordsHaveDistinctRelayKeys: provisioning mints a +// fresh random key per hive. +func TestProvisionedHiveRecordsHaveDistinctRelayKeys(t *testing.T) { + a, b := newWebhookRelayKey(), newWebhookRelayKey() + if len(a) != 64 || len(b) != 64 || a == b { + t.Fatalf("relay keys %q / %q, want two distinct 256-bit hex keys", a, b) + } +} + +// TestHiveStatusNeverExposesRelayKey: the owner-facing hive status returns +// the hive record, but never the relay key, which only the spoke's own +// heartbeat may carry. +func TestHiveStatusNeverExposesRelayKey(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + s := &HubServer{logger: slog.Default(), hubSecret: testHubSecret} + mkUser(t, "alice") + if err := saveSaaSHive(&SaaSHive{ID: "h1", Owner: "alice", WebhookRelayKey: "relay-key-h1"}); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + s.handleHiveStatus(rec, setPathValue(reqWithUser(http.MethodGet, "/st", "", "alice"), "id", "h1")) + if rec.Code != http.StatusOK { + t.Fatalf("owner hive-status = %d, want 200", rec.Code) + } + var got map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if _, ok := got["webhook_relay_key"]; ok || bytes.Contains(rec.Body.Bytes(), []byte("relay-key-h1")) { + t.Fatalf("hive status leaked the relay key: %s", rec.Body.String()) + } + if h := loadSaaSHive("h1"); h == nil || h.WebhookRelayKey != "relay-key-h1" { + t.Fatal("redacting the response must not touch the stored record") + } +} + +// TestWebhookRelayKeyHeartbeatWire: the key the hub attaches reaches the +// spoke's decoder, and the fingerprint the spoke reports reaches the hub's, +// with the same fingerprint format on both sides. +func TestWebhookRelayKeyHeartbeatWire(t *testing.T) { + raw, err := json.Marshal(HeartbeatResponse{GitHubAppConfig: &HeartbeatGitHubAppConfig{WebhookRelayKey: "relay-key-h1"}}) + if err != nil { + t.Fatal(err) + } + var spokeResp spoke.HeartbeatResponse + if err := json.Unmarshal(raw, &spokeResp); err != nil { + t.Fatal(err) + } + if spokeResp.GitHubAppConfig == nil || spokeResp.GitHubAppConfig.WebhookRelayKey != "relay-key-h1" { + t.Fatalf("spoke decoded %+v, want the relay key", spokeResp.GitHubAppConfig) + } + + raw, err = json.Marshal(spoke.HeartbeatPayload{HiveID: "h1", WebhookRelayKeyFingerprint: webhookRelayKeyFingerprint("relay-key-h1")}) + if err != nil { + t.Fatal(err) + } + var hubPayload HeartbeatPayload + if err := json.Unmarshal(raw, &hubPayload); err != nil { + t.Fatal(err) + } + if hubPayload.WebhookRelayKeyFingerprint != webhookRelayKeyFingerprint("relay-key-h1") { + t.Fatalf("hub decoded fingerprint %q", hubPayload.WebhookRelayKeyFingerprint) + } +} + +// meta.json carries the relay key, so it must be unreadable to other local +// users like the hub's other secret stores. A pre-existing world-readable hive +// directory must be tightened on save too. +func TestSaveSaaSHiveKeepsRelayKeyPrivate(t *testing.T) { + useTempSaaSHivesDir(t) + + dir := filepath.Join(saasHivesDir, "legacy") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := saveSaaSHive(&SaaSHive{ID: "legacy", WebhookRelayKey: newWebhookRelayKey()}); err != nil { + t.Fatal(err) + } + + fi, err := os.Stat(filepath.Join(dir, "meta.json")) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm(); got != 0o600 { + t.Fatalf("meta.json mode = %o, want 600", got) + } + di, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if got := di.Mode().Perm(); got != 0o700 { + t.Fatalf("hive dir mode = %o, want 700", got) + } +} + +// Tightening must only strip group/other bits. A hive directory that has +// been made read-only must stay read-only so the save fails loudly instead +// of being silently widened back to writable (which also masked the forced +// save failures in reset_assignment_sweep_branches_test.go). +func TestSaveSaaSHiveNeverWidensReadOnlyDir(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; read-only dir cannot force a write failure") + } + useTempSaaSHivesDir(t) + + dir := filepath.Join(saasHivesDir, "frozen") + if err := os.MkdirAll(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(dir, 0o755) }) + + if err := saveSaaSHive(&SaaSHive{ID: "frozen", WebhookRelayKey: newWebhookRelayKey()}); err == nil { + t.Fatal("saveSaaSHive succeeded into a read-only hive dir; want write failure") + } + di, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if got := di.Mode().Perm(); got != 0o500 { + t.Fatalf("hive dir mode = %o, want 500 (group/other stripped, owner bits untouched)", got) + } +} From 4e3bbc91759274a8fc47006b68201053a09b3a21 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 08:41:05 -0700 Subject: [PATCH 092/223] =?UTF-8?q?=F0=9F=90=9B=20Inbox:=20render=20the=20?= =?UTF-8?q?empty=20state=20instead=20of=20"Loading=20inbox=E2=80=A6"=20for?= =?UTF-8?q?ever=20(#17920)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Inbox: render the empty state instead of "Loading inbox…" forever With no voice knowledge connector configured bootInbox never ran, so GET /api/inbox answered 404 and the loader returned early, leaving the placeholder up. The Inbox service now always boots (zero sources is a valid, empty inbox), the list endpoint returns 200 with an empty list even without a service, and a non-OK response renders a visible error. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🌱 Inbox: reference #17918 in comments Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-inbox-empty-state.md | 1 + src/cmd/hive/boot_inbox.go | 10 +++--- src/cmd/hive/boot_inbox_test.go | 27 ++++++++++++++ src/cmd/hive/boot_knowledge_connectors.go | 5 ++- src/docs/inbox-voice-sources.md | 6 ++++ src/pkg/dashboard/api_inbox.go | 17 ++++++--- src/pkg/dashboard/api_inbox_test.go | 13 +++++-- src/pkg/dashboard/inbox_loader_static_test.go | 36 +++++++++++++++++++ src/pkg/dashboard/static-src/js/inbox.js | 10 ++++-- src/pkg/dashboard/static/index.html | 10 ++++-- 10 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 changelog.d/fixed-inbox-empty-state.md create mode 100644 src/cmd/hive/boot_inbox_test.go create mode 100644 src/pkg/dashboard/inbox_loader_static_test.go diff --git a/changelog.d/fixed-inbox-empty-state.md b/changelog.d/fixed-inbox-empty-state.md new file mode 100644 index 0000000000..d18badbc1f --- /dev/null +++ b/changelog.d/fixed-inbox-empty-state.md @@ -0,0 +1 @@ +- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index b30f16ec2b..d4a3e0808b 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -14,9 +14,10 @@ import ( // inboxSweepTimeout bounds the auto-routing sweep run after a connector sync. const inboxSweepTimeout = 2 * time.Minute -// bootInbox builds the Inbox service when at least one configured knowledge -// connector is a voice source (inbox.sources, default plaud/wisprflow). -// Items are read from the connector vaults under <baseDir>/connectors; +// bootInbox builds the Inbox service. Sources are the configured knowledge +// connectors whose type is a voice source (inbox.sources, default +// plaud/wisprflow); with none configured the service still exists and lists +// nothing, so the dashboard shows the empty state. Items are read from the connector vaults under <baseDir>/connectors; // route state persists at <baseDir>/connector-state/_inbox.json (connector // names cannot start with "_", so it never collides with a connector). func (b *boot) bootInbox(baseDir string) { @@ -31,9 +32,6 @@ func (b *boot) bootInbox(baseDir string) { srcs = append(srcs, b.inboxSource(baseDir, typ)) } } - if len(srcs) == 0 { - return - } token := func(ctx context.Context, _ string) (string, error) { if b.appAuth == nil { return "", nil diff --git a/src/cmd/hive/boot_inbox_test.go b/src/cmd/hive/boot_inbox_test.go new file mode 100644 index 0000000000..f1c38a836f --- /dev/null +++ b/src/cmd/hive/boot_inbox_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// With no knowledge connectors configured the Inbox service must still boot +// so GET /api/inbox returns an empty list, not a 404 (#17918). +func TestBootInboxWithoutConnectors(t *testing.T) { + b := &boot{cfg: &config.Config{}, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + b.bootKnowledgeConnectors(bootKnowledgeDeps{}) + if b.inboxSvc == nil { + t.Fatal("inbox service not built without connectors") + } + items, err := b.inboxSvc.Items(context.Background()) + if err != nil || len(items) != 0 { + t.Fatalf("items = %v %v", items, err) + } + if b.knowledgeConnectors != nil { + t.Fatal("connector syncer should stay nil with no connectors") + } +} diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index 3d9cbdc64f..070f0f9b5e 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -89,6 +89,10 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne // bootKnowledgeConnectors builds and starts the connector syncer. A failure // is logged and leaves connectors off; it never blocks the rest of boot. func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { + // The Inbox always boots, even with no connectors, so GET /api/inbox + // answers 200 with an empty list and the panel renders its empty state + // (#17918) instead of waiting forever on a 404. + b.bootInbox(knowledge.BaseDir()) s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil { b.logger.Warn("knowledge connectors disabled", "error", err) @@ -102,7 +106,6 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { deps.startConnectorSyncer(b.ctx, s) } b.logger.Info("knowledge connectors started", "count", len(b.cfg.Knowledge.Connectors)) - b.bootInbox(knowledge.BaseDir()) } // knowledgeConnectorRuntime exposes the syncer to the dashboard. A nil diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md index 4f71b60ba2..9947215654 100644 --- a/src/docs/inbox-voice-sources.md +++ b/src/docs/inbox-voice-sources.md @@ -95,3 +95,9 @@ See [api-reference/knowledge.md](api-reference/knowledge.md) for `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings` and `POST /api/knowledge/connectors/{name}/auth/complete`. + +`GET /api/inbox` always answers `200`: on a hive with no voice connector +configured it returns an empty `items` list and every source as +`configured: false`, and the Inbox panel shows its empty state with a +"Connect sources" button. A connector added under Knowledge → Integrations +starts syncing after the next hive restart. diff --git a/src/pkg/dashboard/api_inbox.go b/src/pkg/dashboard/api_inbox.go index 1ca6dc14bd..67dfba6281 100644 --- a/src/pkg/dashboard/api_inbox.go +++ b/src/pkg/dashboard/api_inbox.go @@ -60,12 +60,19 @@ func (s *Server) inboxService(w http.ResponseWriter) (*inbox.Service, bool) { } func (s *Server) handleInboxList(w http.ResponseWriter, r *http.Request) { - svc, ok := s.inboxService(w) - if !ok { - return - } ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) defer cancel() + if s == nil || s.deps == nil || s.deps.Inbox == nil { + // No service (older boot path / tests): still a valid, empty inbox — + // the panel must render its empty state, not spin on a 404. + resp := InboxListResponse{Items: []inbox.Item{}, Sources: []InboxSourceStatus{}, Repos: s.inboxRepos(), Settings: inbox.Settings{Rules: []config.InboxRule{}}} + if s != nil && s.deps != nil && s.deps.Config != nil { + resp.Sources = s.inboxSources(ctx, s.deps.Config.Inbox.EffectiveSources()) + } + jsonResponse(w, resp) + return + } + svc := s.deps.Inbox items, err := svc.Items(ctx) resp := InboxListResponse{ Items: items, @@ -115,7 +122,7 @@ func (s *Server) inboxSources(ctx context.Context, types []string) []InboxSource } func (s *Server) inboxRepos() []string { - if s.deps == nil || s.deps.Config == nil { + if s == nil || s.deps == nil || s.deps.Config == nil { return []string{} } repos := append([]string(nil), s.deps.Config.Project.Repos...) diff --git a/src/pkg/dashboard/api_inbox_test.go b/src/pkg/dashboard/api_inbox_test.go index 150f36c188..72f407b2b9 100644 --- a/src/pkg/dashboard/api_inbox_test.go +++ b/src/pkg/dashboard/api_inbox_test.go @@ -79,8 +79,17 @@ func decodeInboxList(t *testing.T, body []byte) InboxListResponse { func TestInboxEndpoints(t *testing.T) { s, deps := apiServer(t) - if rec := doGet(s, "/api/inbox"); rec.Code != http.StatusNotFound { - t.Fatalf("inbox without service = %d, want 404", rec.Code) + // No service: still 200 with an empty list so the panel renders its + // empty state instead of "Loading inbox…" forever (#17918). + rec0 := doGet(s, "/api/inbox") + if rec0.Code != http.StatusOK { + t.Fatalf("inbox without service = %d, want 200: %s", rec0.Code, rec0.Body.String()) + } + if empty := decodeInboxList(t, rec0.Body.Bytes()); empty.Items == nil || len(empty.Items) != 0 || empty.Sources == nil || empty.Settings.Rules == nil { + t.Fatalf("empty inbox = %s", rec0.Body.String()) + } + if rec := doGet(s, "/api/inbox/settings"); rec.Code != http.StatusNotFound { + t.Fatalf("settings without service = %d, want 404", rec.Code) } spek := &inboxTestDest{name: inbox.DestSpekRFC} dibs := &inboxTestDest{name: inbox.DestDibs, err: errors.New("dibs down")} diff --git a/src/pkg/dashboard/inbox_loader_static_test.go b/src/pkg/dashboard/inbox_loader_static_test.go new file mode 100644 index 0000000000..1e4e75185a --- /dev/null +++ b/src/pkg/dashboard/inbox_loader_static_test.go @@ -0,0 +1,36 @@ +package dashboard + +import ( + "os" + "strings" + "testing" +) + +// The Inbox loader must never leave the "Loading inbox…" placeholder up: a +// non-OK /api/inbox response renders an error state, and an OK response +// with zero items renders the empty-state text (#17918). +func TestInboxLoaderHandlesNonOKAndEmpty(t *testing.T) { + src, err := os.ReadFile("static-src/js/inbox.js") + if err != nil { + t.Fatal(err) + } + js := string(src) + start := strings.Index(js, "async function fetchInbox()") + if start < 0 { + t.Fatal("fetchInbox missing") + } + body := js[start : strings.Index(js[start:], "dashboardScheduleWork('inbox'")+start] + if strings.Contains(body, "if (!r.ok) return;") { + t.Fatal("fetchInbox bails out on a non-OK response, leaving the loading placeholder in place") + } + for _, want := range []string{"if (!r.ok) {", "_inboxCache = { items: []", "renderInbox();"} { + if !strings.Contains(body, want) { + t.Fatalf("fetchInbox non-OK branch missing %q:\n%s", want, body) + } + } + for _, want := range []string{"if (data.error) html +=", "Nothing yet. Connect Plaud or another voice source"} { + if !strings.Contains(js, want) { + t.Fatalf("renderInbox missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 9d2233d788..c638c57712 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -26,7 +26,13 @@ async function fetchInbox() { try { const r = await fetch('/api/inbox'); - if (!r.ok) return; + if (!r.ok) { + // Never leave "Loading inbox…" up: a 404/500 becomes a visible error. + const body = await r.json().catch(() => ({})); + _inboxCache = { items: [], sources: [], settings: {}, error: body.error || ('HTTP ' + r.status) }; + renderInbox(); + return; + } _inboxCache = await r.json(); renderInbox(); clearTimeout(_inboxBusyTimer); @@ -113,7 +119,7 @@ html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; } - if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync (a newly added connector starts on the next hive restart).') + '</div>'; else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; panel.innerHTML = html; } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 517974a26c..8a6db5eb16 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46222,7 +46222,13 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> async function fetchInbox() { try { const r = await fetch('/api/inbox'); - if (!r.ok) return; + if (!r.ok) { + // Never leave "Loading inbox…" up: a 404/500 becomes a visible error. + const body = await r.json().catch(() => ({})); + _inboxCache = { items: [], sources: [], settings: {}, error: body.error || ('HTTP ' + r.status) }; + renderInbox(); + return; + } _inboxCache = await r.json(); renderInbox(); clearTimeout(_inboxBusyTimer); @@ -46309,7 +46315,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; } - if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync (a newly added connector starts on the next hive restart).') + '</div>'; else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; panel.innerHTML = html; } From be41521b87dcd918a163e0f8050f34cb03a41bc5 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 08:55:46 -0700 Subject: [PATCH 093/223] =?UTF-8?q?=F0=9F=93=96=20releases:=20candidate=20?= =?UTF-8?q?hives=20follow=20their=20stored=20auto-upgrade=20cadence=20(#17?= =?UTF-8?q?922)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since #11816 the hub respects a candidate hive's daily/weekly mode by default; the docs still said the opposite, which read as a stuck upgrade on a daily hive. Document the actual behaviour and how to switch a hive to instant. Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-docs-candidate-cadence.md | 1 + src/docs/releases.md | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-docs-candidate-cadence.md diff --git a/changelog.d/fixed-docs-candidate-cadence.md b/changelog.d/fixed-docs-candidate-cadence.md new file mode 100644 index 0000000000..9989234640 --- /dev/null +++ b/changelog.d/fixed-docs-candidate-cadence.md @@ -0,0 +1 @@ +- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. diff --git a/src/docs/releases.md b/src/docs/releases.md index 0dc22efe02..704bc9de42 100644 --- a/src/docs/releases.md +++ b/src/docs/releases.md @@ -212,13 +212,18 @@ separate, deliberate policy described in silently couples to it, on purpose — the operator explicitly asked for these to stay decoupled. -Hub-managed hives on the `candidate` channel are canaries: by default the hub -does not hold them behind the daily/weekly auto-upgrade cadence. Once the -candidate image for a newer commit is published, the hub may arm the hive on the -next poll as long as the existing idle, health and wave gates allow it. Stable -channel hives keep the slower soak/cadence rules. Operators who deliberately -want candidate hives to obey their stored cadence can set -`HIVE_CANDIDATE_RESPECT_AUTO_UPGRADE_SCHEDULE=true` on the hub. +Hub-managed hives on the `candidate` channel follow their stored auto-upgrade +cadence (`auto_upgrade_mode`: `daily` by default, `weekly`, or `instant`) just +like stable hives (#11816). A daily candidate hive therefore picks up the newest +published candidate image once a day, in the first poll at or after 1pm ET; the +hub reports the hold as `upgradeWhyNotNow: "before the daily window"` and the +spoke's `/api/version` shows `state: behind` until then. To track every +published candidate image, set the hive's mode to `instant` +(`PUT /api/saas/hives/{id}/auto-upgrade` with +`{"auto_upgrade": true, "auto_upgrade_mode": "instant"}`, or the Manage → +Auto-upgrade control on the hub). Setting +`HIVE_CANDIDATE_RESPECT_AUTO_UPGRADE_SCHEDULE=false` on the hub restores the +old behaviour where every candidate hive bypasses its cadence. The hub itself does **not** follow the hosted-hive daily/weekly cadence: when hub auto-upgrade is enabled and not paused, the hub rolls itself **instantly** From 3b017982d1023709dc4b1973ddeda3a827064fb4 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 16:00:30 +0000 Subject: [PATCH 094/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.235.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.235.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-baseline-check-gh-wrapper.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-baseline-check-gh-wrapper.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0337292185..bd5e2eb51d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.235.1) + +### Fixed + +- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. + ## 2026-11-12 (v5.235.0) ### Added diff --git a/changelog.d/fixed-baseline-check-gh-wrapper.md b/changelog.d/fixed-baseline-check-gh-wrapper.md deleted file mode 100644 index 4d3ccfe69b..0000000000 --- a/changelog.d/fixed-baseline-check-gh-wrapper.md +++ /dev/null @@ -1 +0,0 @@ -- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. From c16b0279c0502df78a2e933d1355c3a5049d1487 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 16:30:20 +0000 Subject: [PATCH 095/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 16 ++++++++++++++++ changelog.d/added-claude-long-lived-token.md | 1 - ...ed-claude-operator-token-suspends-rebridge.md | 1 - changelog.d/fixed-docs-candidate-cadence.md | 1 - .../fixed-hosted-webhook-relay-signature.md | 1 - changelog.d/fixed-inbox-empty-state.md | 1 - 6 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-claude-long-lived-token.md delete mode 100644 changelog.d/changed-claude-operator-token-suspends-rebridge.md delete mode 100644 changelog.d/fixed-docs-candidate-cadence.md delete mode 100644 changelog.d/fixed-hosted-webhook-relay-signature.md delete mode 100644 changelog.d/fixed-inbox-empty-state.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bd5e2eb51d..c1159587c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.0) + +### Added + +- Claude agents can run on an operator-supplied long-lived token: set `HIVE_CLAUDE_OAUTH_TOKEN` (or mount it at `/secrets/claude-oauth-token`, or point `HIVE_CLAUDE_OAUTH_TOKEN_FILE` at it) to a `claude setup-token` token and every `claude`-backend agent is launched with it as `CLAUDE_CODE_OAUTH_TOKEN`, ahead of the 8-hour dashboard sign-in; unset, nothing changes. + +### Changed + +- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) + +### Fixed + +- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. +- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. +- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. + ## 2026-11-12 (v5.235.1) ### Fixed diff --git a/changelog.d/added-claude-long-lived-token.md b/changelog.d/added-claude-long-lived-token.md deleted file mode 100644 index 9c84edba5b..0000000000 --- a/changelog.d/added-claude-long-lived-token.md +++ /dev/null @@ -1 +0,0 @@ -- Claude agents can run on an operator-supplied long-lived token: set `HIVE_CLAUDE_OAUTH_TOKEN` (or mount it at `/secrets/claude-oauth-token`, or point `HIVE_CLAUDE_OAUTH_TOKEN_FILE` at it) to a `claude setup-token` token and every `claude`-backend agent is launched with it as `CLAUDE_CODE_OAUTH_TOKEN`, ahead of the 8-hour dashboard sign-in; unset, nothing changes. diff --git a/changelog.d/changed-claude-operator-token-suspends-rebridge.md b/changelog.d/changed-claude-operator-token-suspends-rebridge.md deleted file mode 100644 index 22c625b98c..0000000000 --- a/changelog.d/changed-claude-operator-token-suspends-rebridge.md +++ /dev/null @@ -1 +0,0 @@ -- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) diff --git a/changelog.d/fixed-docs-candidate-cadence.md b/changelog.d/fixed-docs-candidate-cadence.md deleted file mode 100644 index 9989234640..0000000000 --- a/changelog.d/fixed-docs-candidate-cadence.md +++ /dev/null @@ -1 +0,0 @@ -- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. diff --git a/changelog.d/fixed-hosted-webhook-relay-signature.md b/changelog.d/fixed-hosted-webhook-relay-signature.md deleted file mode 100644 index 436f4aa770..0000000000 --- a/changelog.d/fixed-hosted-webhook-relay-signature.md +++ /dev/null @@ -1 +0,0 @@ -- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. diff --git a/changelog.d/fixed-inbox-empty-state.md b/changelog.d/fixed-inbox-empty-state.md deleted file mode 100644 index d18badbc1f..0000000000 --- a/changelog.d/fixed-inbox-empty-state.md +++ /dev/null @@ -1 +0,0 @@ -- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. From 9a41695691e84875e40976bfedd03cb22249f312 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:56:34 +0000 Subject: [PATCH 096/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20ADR-0026=20?= =?UTF-8?q?=E2=80=94=20Substrate=20actors=20run=20credential-free;=20hive?= =?UTF-8?q?=20brokers=20pushes=20post-kick=20(#17934)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: ADR-0026 — Substrate actors run credential-free; hive brokers pushes post-kick Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * changelog: fragment for #17934 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-17934.md | 1 + src/deploy/substrate.md | 4 ++-- .../adr/0026-sandbox-runtime-selection.md | 21 ++++++++++++++++++- src/docs/sandbox-isolation.md | 2 +- 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 changelog.d/changed-17934.md diff --git a/changelog.d/changed-17934.md b/changelog.d/changed-17934.md new file mode 100644 index 0000000000..f0e9d94f58 --- /dev/null +++ b/changelog.d/changed-17934.md @@ -0,0 +1 @@ +- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) diff --git a/src/deploy/substrate.md b/src/deploy/substrate.md index 941d32d2c3..707db2e458 100644 --- a/src/deploy/substrate.md +++ b/src/deploy/substrate.md @@ -84,8 +84,8 @@ Then create the pieces hive's runtime depends on, in Substrate terms: it is tracked in [#17697](https://github.com/hivecommons/hive/issues/17697). - **Egress**: hive writes its own `EgressPolicy` per actor (`github.com`, `api.github.com`, `allow_hosts`). Do **not** also configure Substrate's - credential injection for GitHub — hive's MITM proxy brokers credentials and - the two would conflict. + credential injection for GitHub — actors run credential-free and hive + brokers pushes after the kick (ADR-0026, #17695). ## Verify diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index d5f27798c2..aae4f451f1 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -242,7 +242,7 @@ opt-in and inert unless selected. |---|---|---| | Kernel-enforced confinement | gVisor user-space kernel or a microVM; worker pods are `privileged: false` but keep `SYS_ADMIN`/`NET_ADMIN` | selects the class through `worker_selector`; no change to hive | | Default-deny egress | `atenet-egress` `EgressPolicy` per actor; rules are TLS-passthrough or terminate per host:port, no per-method read-only mode | `SubstrateEgressPolicy`: `github.com` and `api.github.com` on 443 plus `sandbox.substrate.allow_hosts`, TLS passthrough; everything else (and all UDP) denied | -| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | unchanged hive brokering — the agent still gets `HTTPS_PROXY` to the hive MITM proxy (ADR-0002); Substrate's injection is **not** used, the two are complementary | +| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | actors run credential-free: no `HTTPS_PROXY`/MITM broker inside the actor and no GitHub credential enters it; hive does every push and GitHub write after the kick; Substrate's injection is **not** used (see the amendment below) | | Workspace in, commits out | `durableDir` volumes and object-storage snapshots; no exec/attach API | an in-actor HTTP **bridge** (`crux substrate-bridge`) reached through the router's mTLS `CONNECT` tunnel: push tar → `POST /run` → pull tar; the hive-side workspace stays authoritative | | Cleanup and timeouts | `DeleteActor` / `SuspendActor` / `DeleteActorTemplate` | one template + actor per kick, both deleted on success, failure and timeout (fresh bounded context, `ExitCode -1`, 16 MiB cap, as `job`/`openshell`) | @@ -257,6 +257,25 @@ opt-in and inert unless selected. | snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; only the latest suspended actor per agent per hive process is retained (the previous one is deleted on the next suspend; actors from before a restart are not reclaimed); resume-on-next-kick is not implemented (#17697) | | `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | +### Amendment (2026-11-12, #17695): credential-free actors + +Decision by the hive owner on #17695, option A: + +- Substrate actors run **credential-free**. There is no `HTTPS_PROXY` or MITM + broker inside the actor and no GitHub credential enters it; the actor's + egress is governed only by Substrate's `EgressPolicy`. +- Hive brokers pushes **post-kick**: it performs git pushes and GitHub writes + itself, from the workspace the bridge returns. +- **Phase 3 (future)**: SVID-authenticated broker delegation over the atenet + mTLS channel, so hive can delegate scoped GitHub writes to the actor without + a credential entering it. It is gated on hosted clusters serving the podcert + feature gates (`PodCertificateRequest` / `ClusterTrustBundle`, + `certificates.k8s.io/v1beta1`) and is tracked as a child issue of #17695. + +This supersedes the earlier statement that hive's credential brokering is +unchanged for Substrate. The podman, `job` and `openshell` runtimes keep +their proxy model (ADR-0002). + ### Cluster prerequisites and which hive clusters qualify Substrate's real requirement is **not** a kernel floor: the gVisor class runs diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index a1e1779719..17cdbca242 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -319,7 +319,7 @@ agents: **One actor per kick.** Hive creates an `ActorTemplate` (one container running the bridge command, the workspace as a `durableDir`, the operator's limits and `worker_selector`) and an `Actor` from it, both named `hive-kick-<agent>-<unix>`, attaches the egress policy, waits for the actor to report `RUNNING`, and then talks to a small HTTP **bridge** inside the actor (`crux substrate-bridge`, overridable with `bridge_command`, port `bridge_port`) through the router's mTLS `CONNECT` tunnel: it pushes the workspace as a tar, runs the kick with the sanitized environment, and pulls the workspace back. The hive-side workspace remains the source of truth and feeds the push broker exactly as for Podman. Substrate has no exec API and hive shares no PVC with the worker nodes, which is why the workspace is copied rather than mounted; the copy is bounded by the kick timeout. -**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Credential brokering is unchanged: the agent inside the actor still receives `HTTPS_PROXY` pointing at the hive MITM proxy ([ADR-0002](adr/0002-mitm-proxy-network-enforcement.md)); Substrate's own zero-credential injection is complementary and not used by hive. +**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Actors run credential-free: there is no `HTTPS_PROXY` or MITM broker inside the actor and no credential enters it, and hive brokers pushes after the kick from the returned workspace (decision on [#17695](https://github.com/hivecommons/hive/issues/17695), [ADR-0026](adr/0026-sandbox-runtime-selection.md)). Substrate's own credential injection is not used; SVID-authenticated broker delegation over the atenet mTLS channel is a future phase 3. **Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). At most one suspended actor is retained per agent per hive process: the previous one is deleted after the next successful suspend (skipped with a warning if the agent's `atespace` changed in between), and actors left by an earlier hive process are not reclaimed because the Control API has no `ListActors`. `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. From d4a1c58a7e9cdc6f3a6b550a6df4cd56c29949f1 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:13:05 +0100 Subject: [PATCH 097/223] =?UTF-8?q?=F0=9F=90=9B=20Classify=20missing=20Dep?= =?UTF-8?q?endabot=20alerts=20permission=20(#17929)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Classify missing Dependabot alerts permission Hive-Run: hivecommons/hive#17925 Hive-Plan: classify missing Dependabot alerts permission Hive-Spec: fleet-report#runtime-error-classes Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * 🐛 Fix Dependabot report changelog category Hive-Run: hivecommons/hive#17925 Hive-Plan: classify missing Dependabot alerts permission Hive-Spec: fleet-report#runtime-error-classes Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * test: cover Dependabot permission runtime thresholds Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --- .../fixed-dependabot-permission-selfreport.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/delegatedecision/issue_judgment.go | 31 ++++++++++--------- .../delegatedecision/issue_judgment_test.go | 9 ++++++ src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 2 ++ src/pkg/fleetreport/slogfeed/slogfeed_test.go | 20 ++++++++++++ 7 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-dependabot-permission-selfreport.md diff --git a/changelog.d/fixed-dependabot-permission-selfreport.md b/changelog.d/fixed-dependabot-permission-selfreport.md new file mode 100644 index 0000000000..2438213269 --- /dev/null +++ b/changelog.d/fixed-dependabot-permission-selfreport.md @@ -0,0 +1 @@ +- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index fcd9a05adc..fdc732371d 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -145,6 +145,7 @@ title is `[selfreport] <class>: <short message>`. | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | +| `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index 377adbc3da..4d1ea1a596 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -94,21 +94,22 @@ func ParseSelfReport(title, body string) (SelfReport, bool) { // constants (pinned by TestSelfreportOpsClassesMatchFleetReport) without // importing that package's dependency tree into this leaf. var selfreportOpsRemedies = map[string]string{ - "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", - "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", - "data_disk_pressure": "free or grow the data disk on the spoke", - "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", - "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", - "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", - "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", - "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", - "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", - "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", - "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", - "inference_upstream_error": "check the inference endpoint the spoke's provider points at", - "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", - "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", - "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", + "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", + "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", + "data_disk_pressure": "free or grow the data disk on the spoke", + "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", + "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", + "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", + "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", + "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", + "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", + "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", + "inference_upstream_error": "check the inference endpoint the spoke's provider points at", + "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", + "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", + "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", } // selfreportOpsClass reports whether a self-report class is an operator's to diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 9f0bdc0ae3..6e17ebf4ca 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -27,6 +27,14 @@ func TestDecideIssueLikeTheOwner(t *testing.T) { text: []string{"@clubanderson", "re-authenticate the agent provider", "hosted-ibm-alchemy-logg-ptoo"}, actions: []ActionKind{ActionLabelsAdd, ActionRecommendOnly}, }, + { + name: "missing Dependabot alerts permission is operator action", + in: Input{Title: "[selfreport] github_app_alerts_permission_missing: scoped token mint", + Body: "<!-- hive:selfreport fp=x hive=spoke -->\nclass `github_app_alerts_permission_missing`\n| owner(s) | @clubanderson |\n| spoke | spoke |"}, + rule: "selfreport-operator-action", verdict: VerdictOperatorAction, + text: []string{"@clubanderson", "Dependabot alerts read-only", "re-accept", "spoke"}, + actions: []ActionKind{ActionLabelsAdd, ActionRecommendOnly}, + }, { name: "hive#15329 RFC on hive-filed proposals (human author) → recommend only, with a real recommendation", in: fixtureHive15329, rule: "hive-policy", verdict: VerdictRecommendOnly, @@ -160,6 +168,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 3946d8adb4..5e56884fd9 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -77,6 +77,7 @@ const ( ClassBeadFactIngestFailed = "bead_fact_ingest_failed" ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" @@ -201,6 +202,7 @@ var Classes = map[string]ClassSpec{ ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, + ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index bf9c2dafbf..ec5791dc89 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -453,6 +453,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, + {class: ClassGitHubAppAlertsPermission, message: "scoped token mint with vulnerability_alerts:read failed; retrying without it"}, {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, @@ -1240,6 +1241,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "fleet report: cleared comment failed": ClassSelfReportClearedFailed, "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 4d2757e831..b92640d6c3 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -335,6 +335,26 @@ func TestRequiredChecksOverrideMismatchLogsWithoutFleetReport(t *testing.T) { } } +func TestMissingAlertsPermissionReportsOperatorCondition(t *testing.T) { + now := time.Date(2026, 11, 12, 15, 40, 0, 0, time.UTC) + tr := fleetreport.NewRuntimeTracker(func() time.Time { return now }) + var out bytes.Buffer + logger := slog.New(NewWithObserver(slog.NewJSONHandler(&out, nil), tr.Observe)) + const msg = "scoped token mint with vulnerability_alerts:read failed; retrying without it — the sec-check Dependabot alerts call will 403 until the App grants Dependabot alerts read-only and the installation re-accepts it; see docs/github-app-setup.md" + for range 5 { + logger.Warn(msg, "tier", "sec-check", "error", "permissions requested are not granted") + now = now.Add(time.Minute) + } + ev := tr.Evidence() + if len(ev) != 1 || ev[0].Class != fleetreport.ClassGitHubAppAlertsPermission || ev[0].Count != 5 { + t.Fatalf("missing permission evidence = %+v", ev) + } + now = now.Add(21 * time.Minute) + if ev := tr.Evidence(); len(ev) != 0 { + t.Fatalf("quiet condition did not clear: %+v", ev) + } +} + func TestFeedLogFedPath(t *testing.T) { tests := []struct { name string From 1487e96d8c671947df3cd2df4c92915d80e4f899 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 17:29:59 +0000 Subject: [PATCH 098/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/changed-17934.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/changed-17934.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c1159587c9..73d850f252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.1) + +### Changed + +- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) + ## 2026-11-12 (v5.236.0) ### Added diff --git a/changelog.d/changed-17934.md b/changelog.d/changed-17934.md deleted file mode 100644 index f0e9d94f58..0000000000 --- a/changelog.d/changed-17934.md +++ /dev/null @@ -1 +0,0 @@ -- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) From 435c8653d16e240b82c40ffd89024ba691590e27 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:49:29 +0100 Subject: [PATCH 099/223] =?UTF-8?q?=F0=9F=90=9B=20Handle=20unsupported=20s?= =?UTF-8?q?igned-commit=20rewrites=20explicitly=20(#17927)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hive-Run: hivecommons/hive#17923 Hive-Plan: signed-commit-reconciler-refusal Hive-Spec: hivecommons/hive#17923 Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-signed-reconcile-unsupported.md | 1 + src/docs/github-app-setup.md | 4 +- src/pkg/github/pr_signed_reconcile.go | 32 ++++++++------ src/pkg/github/pr_signed_reconcile_test.go | 42 +++++++++++++++++++ 4 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 changelog.d/fixed-signed-reconcile-unsupported.md diff --git a/changelog.d/fixed-signed-reconcile-unsupported.md b/changelog.d/fixed-signed-reconcile-unsupported.md new file mode 100644 index 0000000000..ad1001d019 --- /dev/null +++ b/changelog.d/fixed-signed-reconcile-unsupported.md @@ -0,0 +1 @@ +- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 2976b95c7b..0bf7f5d43a 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -257,7 +257,9 @@ What it does not do, and falls back on (the PR still opens on the agent's own co **Follow-up commits.** An agent's later pushes to its open PR (a CI fix, review follow-ups, a rebase) are plain git and unsigned, and so is a PR that opened on a `signed_skipped` fallback. With `app_signed_commits` on, the PR-request watcher also runs a reconcile pass, at most once a minute (#9364). A PR needs signing when *any* of its commits is not **Verified**, not only its head: a person can sign their own commit on top of an agent's unsigned one, which leaves the head Verified while the PR is still blocked (#9531). For every open PR authored by the App bot, the pass finds the first unverified commit (oldest to newest) and re-authors everything from there through the head as one GitHub-signed commit on top of the commit right before it. The tree is the same, and the message carries the range's messages and DCO trailers, re-addressed as above. When the PR's first commit is the unverified one, it signs against the PR's base branch, as at open time. A fully Verified PR is left alone. The open path's guards all still apply. A head that moves mid-rewrite is retried on the next pass, and so is a head GitHub could not be reached for (a 5xx or network error). A head GitHub *refuses* to sign (a 4xx answer, such as the 422 on the scratch ref in #16193) is retried on three consecutive passes and then settled, with the `hive:signed-commits-blocked` comment quoting the error; there is no person's commit to squash in that case, so the remedy is to push a new head (which restarts signing) or fix the repository-side cause the error names. A PR whose head hasn't changed since the last pass costs no calls beyond the open-PR list. -The pass re-signs only the hive's own work. Every commit in that range, Verified or not, must have an author and committer that are the App bot or an address in the pane identity domain (`HIVE_GIT_BOT_EMAIL_DOMAIN`, default `hive.kubestellar.io`). Verified App-bot commits in the range are folded in. A person's commit (even a Verified one on top), or a merge commit, leaves the branch as pushed. So does any change the mutation can't express. The PR then gets one comment (marked `<!-- hive:signed-commits-blocked -->`, posted once per PR) naming the unsigned commit and the commit that blocks the rewrite, so it can't merge under `required_signatures` until someone acts. The manual fix is to squash the unsigned commit into a commit the person signs, or to drop the person's commit so the hive can re-sign the range and then push it again, signed, on top. +The pass re-signs only the hive's own work. Every commit in that range, Verified or not, must have an author and committer that are the App bot or an address in the pane identity domain (`HIVE_GIT_BOT_EMAIL_DOMAIN`, default `hive.kubestellar.io`). Verified App-bot commits in the range are folded in. A person's commit (even a Verified one on top), or a merge commit, leaves the branch as pushed. The PR gets one comment (marked `<!-- hive:signed-commits-blocked -->`, posted once per PR) naming the blocker. To unblock it, squash the unsigned commit into a commit the person signs, or drop the person's commit so the hive can re-sign the range and then push it again, signed, on top. + +Changes the signing mutation cannot safely express also leave the branch untouched and get a single comment with the reason. For example, it cannot preserve executable files, symlinks or submodule pointers. A contributor can re-author the unsigned commits with their own verified signature, preserving their contents and modes, and push the signed branch. These deliberate skips are logged at INFO rather than as API failures; a new head is examined again. **Agents after a push.** The rewrite replaces the agent's commits on the remote, so its local branch no longer matches, even though the tree is identical. Before committing again on a branch with an open PR, run `git fetch && git rebase origin/<branch>`. The rebase drops the now-empty local commits and keeps any new work. Never force-push over the signed commit: that puts the unsigned commits back, and the next pass has to sign them again. diff --git a/src/pkg/github/pr_signed_reconcile.go b/src/pkg/github/pr_signed_reconcile.go index e2a999337f..b8770a30bd 100644 --- a/src/pkg/github/pr_signed_reconcile.go +++ b/src/pkg/github/pr_signed_reconcile.go @@ -69,6 +69,12 @@ const signedSkipRemedyPerson = "The hive re-signs only its own agents' commits ( "or drop your own commit from the branch so the hive can re-sign the agent's work on a later pass, " + "then push your change again, signed, on top." +// signedSkipRemedyUnsupported applies when the change cannot safely be +// re-authored, rather than a person's commit blocking the rewrite. +const signedSkipRemedyUnsupported = "The hive could not safely represent this change with GitHub's signing API. " + + "Have a contributor re-author the unsigned commits with their own verified signature and push the signed branch, " + + "preserving the file modes and contents. A new head also lets the hive try again." + // signedSkipRemedyAPIRefused tells the PR owner how to unblock a head GitHub // kept refusing to sign: there is no person's commit to squash, only a new // head (or a repository-side fix) re-arms the reconciler. @@ -336,23 +342,25 @@ func (c *Client) reconcileSignedPR(ctx context.Context, owner, repo string, pr * oid, replaced, err := c.signBranch(ctx, owner, repo, base, head) if err != nil { var moved signedHeadMovedError - c.warn("signed-commit reconciler: could not re-author the unsigned tail as a GitHub-signed commit", append(logArgs, "base", base, "reason", err.Error())...) if errors.As(err, &moved) { + c.info("signed-commit reconciler: head moved while signing; will retry", append(logArgs, "base", base, "reason", err.Error())...) return "" // transient: retry next pass } - if !errors.Is(err, errSignedCommitSkip) { - if !signedAPIRefusal(err) { - return "" // outage (5xx, network): retry next pass, uncounted - } - if n := c.signedReconcile.noteFailure(key, headSHA); n < signedReconcileMaxAttempts { - return "" // retry next pass - } - c.warn("signed-commit reconciler: giving up on this head after repeated refusals", append(logArgs, "attempts", signedReconcileMaxAttempts)...) - reason := fmt.Sprintf("GitHub refused to sign this head %d times in a row: %s", signedReconcileMaxAttempts, signedSkipReason(err)) - c.noteSignedSkip(ctx, owner, repo, number, reason, signedSkipRemedyAPIRefused) + if errors.Is(err, errSignedCommitSkip) { + c.info("signed-commit reconciler: cannot safely re-author unsigned tail; leaving branch unchanged", append(logArgs, "base", base, "reason", err.Error())...) + c.noteSignedSkip(ctx, owner, repo, number, err.Error(), signedSkipRemedyUnsupported) return headSHA } - c.noteSignedSkip(ctx, owner, repo, number, err.Error(), signedSkipRemedyPerson) + c.warn("signed-commit reconciler: could not re-author the unsigned tail as a GitHub-signed commit", append(logArgs, "base", base, "reason", err.Error())...) + if !signedAPIRefusal(err) { + return "" // outage (5xx, network): retry next pass, uncounted + } + if n := c.signedReconcile.noteFailure(key, headSHA); n < signedReconcileMaxAttempts { + return "" // retry next pass + } + c.warn("signed-commit reconciler: giving up on this head after repeated refusals", append(logArgs, "attempts", signedReconcileMaxAttempts)...) + reason := fmt.Sprintf("GitHub refused to sign this head %d times in a row: %s", signedReconcileMaxAttempts, signedSkipReason(err)) + c.noteSignedSkip(ctx, owner, repo, number, reason, signedSkipRemedyAPIRefused) return headSHA } c.signedReconcile.clearFailures(key) diff --git a/src/pkg/github/pr_signed_reconcile_test.go b/src/pkg/github/pr_signed_reconcile_test.go index 4245f5bd57..9b65a47495 100644 --- a/src/pkg/github/pr_signed_reconcile_test.go +++ b/src/pkg/github/pr_signed_reconcile_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "net/http" "strings" "testing" @@ -396,6 +397,47 @@ func TestSignedReconcile_UnchangedHeadIsNotRelisted(t *testing.T) { } } +// Unsupported file modes are a deliberate safety refusal, not an API outage. +// Keep the branch intact, explain how to unblock it, and settle the head. +func TestSignedReconcile_UnsupportedModeSettlesWithActionableNote(t *testing.T) { + for _, mode := range []string{"100755", "120000", "160000"} { + t.Run(mode, func(t *testing.T) { + m, c := reconcileFixture(t, true) + m.treeMode["tests/new_test.py"] = mode + m.prCommits = []map[string]any{prCommitJSON(m.headSHA, false, "", "quality@hive.kubestellar.io")} + var logs strings.Builder + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + + c.reconcileSignedCommits(context.Background()) + c.reconcileSignedCommits(context.Background()) + + if m.refPatch != nil || m.graphql != nil || m.callIndex("POST /repos/o/r/git/refs") >= 0 { + t.Fatalf("unsupported mode must leave the branch untouched: %v", m.calls) + } + if m.countCalls(prCommitsCall) != 1 { + t.Errorf("unchanged unsupported head must be settled: %v", m.calls) + } + if len(m.posted) != 1 || !strings.Contains(m.posted[0], mode) || + !strings.Contains(m.posted[0], "their own verified signature") || + strings.Contains(m.posted[0], "drop your own commit") { + t.Fatalf("expected one actionable mode-specific comment, got %q", m.posted) + } + if strings.Contains(logs.String(), "could not re-author the unsigned tail") { + t.Errorf("expected skip must not be reported as an API failure: %s", logs.String()) + } + + newHead := "dddd000000000000000000000000000000000030" + m.headSHA = newHead + m.openPRs = []map[string]any{reconcilePR(77, reconcileBot, newHead)} + m.prCommits = []map[string]any{prCommitJSON(newHead, false, "", "quality@hive.kubestellar.io")} + c.reconcileSignedCommits(context.Background()) + if m.countCalls(prCommitsCall) != 2 || len(m.posted) != 1 { + t.Errorf("new head must be retried without another comment: calls=%v posted=%q", m.calls, m.posted) + } + }) + } +} + // A head that moved while the signed commit was built is transient: no // comment, and the next pass looks again (the open-time skip is retried). func TestSignedReconcile_MovedHeadRetriesNextPass(t *testing.T) { From a6c1b36179f6b7ca9fb8a92b25ad8971e1e211ae Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:02:36 +0100 Subject: [PATCH 100/223] =?UTF-8?q?=F0=9F=90=9B=20Avoid=20fleet=20reports?= =?UTF-8?q?=20for=20canceled=20adoption=20reads=20(#17931)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hive-Run: hivecommons/hive#17926 Hive-Plan: classify canceled adoption sweep reads Hive-Spec: fleet-report#runtime-error-classes Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --- changelog.d/fixed-canceled-adoption-report.md | 1 + src/docs/fleet-report.md | 4 ++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 21 ++++++++++++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 18 ++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-canceled-adoption-report.md diff --git a/changelog.d/fixed-canceled-adoption-report.md b/changelog.d/fixed-canceled-adoption-report.md new file mode 100644 index 0000000000..803255f554 --- /dev/null +++ b/changelog.d/fixed-canceled-adoption-report.md @@ -0,0 +1 @@ +- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index fdc732371d..8a3f887968 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -175,6 +175,10 @@ when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. Issue activity keeps its last rollup and is retried by the collector. +Canceled maintainer-edit access reads during an in-place adoption sweep stay in +local logs without triggering the catch-all; adoption is preserved and the +read is retried on the next sweep. Other access-read errors remain reportable. + Callers that fail closed on a GitHub HTTP 500 from the combined-status endpoint — the merge-eligible lane gate, the trusted-author and contributor-lane automerge sweeps, and the stage-label check digest — also stay in the logs diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index a5ee890b10..1550bafe9b 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -14,6 +14,7 @@ package slogfeed import ( "context" + "errors" "log/slog" "strings" @@ -114,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledAdoptionRead(r) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -135,6 +136,24 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } +// A canceled sweep leaves the existing adoption intact and retries later. +// Other failures reading push access still warrant a repeated-error report. +func canceledAdoptionRead(r slog.Record) bool { + if strings.TrimSpace(r.Message) != "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" { + return false + } + canceled := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "error" { + err, ok := a.Value.Any().(error) + canceled = (ok && errors.Is(err, context.Canceled)) || a.Value.String() == context.Canceled.Error() + return !canceled + } + return true + }) + return canceled +} + // expectedKickHold reports whether a "failed to send kick" record carries the // deliberate post-restart kick cooldown refusal (#7363). The hold is the // breaker working as designed, not a repeated runtime defect (#14422); other diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index b92640d6c3..ce7c2d09ab 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -64,6 +64,24 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestCanceledAdoptionReadStaysInLog(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" + for range 8 { + logger.Warn(msg, "repo", "server", "pr", 649, "error", &url.Error{Op: "GET", URL: "https://api.github.com", Err: context.Canceled}) + } + logger.Warn(msg, "error", "context canceled") + if len(*seen) != 0 || strings.Count(out.String(), "\n") != 9 { + t.Fatalf("canceled reads: observations=%+v, log=%s", *seen, out.String()) + } + logger.Warn(msg, "error", errors.New("permission denied")) + logger.Warn(msg, "error", context.DeadlineExceeded) + logger.Warn("another operation failed", "error", context.Canceled) + if len(*seen) != 3 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + // #16510: polling an exhausted rebase episode must keep warning locally // without turning the intentional handoff into a repeated runtime defect. func TestRebaseAttemptsExhaustedLogsWithoutFleetReport(t *testing.T) { From d17359e507ecce6213b365a1463822a70bc643d3 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 11:12:02 -0700 Subject: [PATCH 101/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20transpor?= =?UTF-8?q?t-level=20App=20token=20mint=20failures=20as=20forge=5Funreacha?= =?UTF-8?q?ble,=20not=20a=20mint=20failure=20(#17948)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: classify transport-level token mint failures as forge_unreachable A hosted GHE hive on a cluster whose egress to github.ibm.com was blocked filed github_app_token_mint_failed (high, github-client) for every agent, and the Diagnostics remedy told the owner to check the App key and installation — which were fine. The mint never reached GitHub. mintAgentTokenUnlocked now checks the mint error with the same test the dashboard forge probe uses (#17427): context deadline, *url.Error or net.Error → observe crux.ClassForgeUnreachable (component network, medium) and clear the mint-failed condition; other errors keep the mint-failed class and clear forge_unreachable; success clears all three. Documented in docs/fleet-report.md; selfreport remedy points at egress, DNS and proxy/hostAliases. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 fix: move mint classification to manager_mint.go (crux size ratchet) and add forge_unreachable threshold case Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 test: register forge_unreachable in the selfreport ops-class mirror Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-forge-unreachable-mint-class.md | 1 + src/crux/agent/manager_mint.go | 98 +++++++++++++++++++ src/crux/agent/manager_tokens.go | 55 ----------- src/crux/agent/mint_forge_unreachable_test.go | 93 ++++++++++++++++++ src/crux/conditions.go | 13 ++- src/docs/fleet-report.md | 1 + src/pkg/delegatedecision/issue_judgment.go | 1 + .../delegatedecision/issue_judgment_test.go | 2 +- src/pkg/fleetreport/runtime.go | 2 + src/pkg/fleetreport/runtime_test.go | 1 + 10 files changed, 208 insertions(+), 59 deletions(-) create mode 100644 changelog.d/fixed-forge-unreachable-mint-class.md create mode 100644 src/crux/agent/manager_mint.go create mode 100644 src/crux/agent/mint_forge_unreachable_test.go diff --git a/changelog.d/fixed-forge-unreachable-mint-class.md b/changelog.d/fixed-forge-unreachable-mint-class.md new file mode 100644 index 0000000000..65050c8d68 --- /dev/null +++ b/changelog.d/fixed-forge-unreachable-mint-class.md @@ -0,0 +1 @@ +- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. diff --git a/src/crux/agent/manager_mint.go b/src/crux/agent/manager_mint.go new file mode 100644 index 0000000000..fe8896f327 --- /dev/null +++ b/src/crux/agent/manager_mint.go @@ -0,0 +1,98 @@ +package agent + +import ( + "context" + "errors" + "net" + "net/url" + + "github.com/hivecommons/hive/crux" +) + +// mintAgentTokenUnlocked mints the per-agent tier-scoped GitHub App token +// (WriteAgentToken) and the opt-in short-lived mint token for the agent. It is +// the single mint step every (re)launch path — Start, Resume, Restart, +// RestartWithBootstrap — runs before launchInTmux (#3962: previously only +// Start minted, so an agent relaunched via Resume/Restart never got a token +// until the whole process restarted; its cache file stayed 0 bytes and gh/git +// push failed silently). +// +// MUST be called with m.mu RELEASED. WriteAgentToken makes an outbound GitHub +// API call (through the MITM egress proxy, which the production incident +// showed can hang), and issueAgentMintToken can do the same. Holding m.mu +// across a hang stalls AllStatuses()/the heartbeat for the WHOLE fleet — the +// exact liveness flap Start's phase split was built to avoid. The function +// reads only immutable agent identity (Name, UID) and Config via agentMode, +// and writes no m.mu-guarded AgentProcess field, so running it lock-free is +// race-free. m.appAuth is read without the lock exactly as Start's unlocked +// phase always did: it is injected once at wiring time, before agents start. +func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProcess) { + if m.appAuth == nil || !m.agentTokenCacheEnabled(agent) { + return + } + tier := m.agentTokenTier(agent) + if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { + // Static misconfiguration: every tick would fail identically, so log + // once per agent until a mint succeeds instead of every supervisor tick. + if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { + m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", + "agent", agent.Name, "tier", tier, "error", err) + } + m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") + } else if err != nil && isForgeTransportErr(err) { + // The GitHub host did not answer (dial timeout, connection refused, + // deadline exceeded). Nothing about the App key or installation can + // be concluded from that, so report it as a forge-reachability + // condition rather than a mint failure — the latter tells the + // operator to re-check App permissions, which is wrong during an + // egress outage. The mint-failed condition is cleared so a hive that + // slid from a real mint error into an outage does not hold both open. + m.logger.Warn("per-agent scoped token NOT delivered — GitHub host unreachable from the pod; this agent's GitHub writes (gh + git push) will FAIL until egress is restored", + "agent", agent.Name, "tier", tier, "error", err) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) + m.observeCondition(crux.ClassForgeUnreachable, agent.Name, "github host unreachable during token mint: "+err.Error(), "") + } else if err != nil { + // Be precise about the blast radius. Since audit H3 the shared-cache + // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no + // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the + // FULL installation token, now owner-only 0600). They FAIL LOUD when + // the per-agent scoped token is absent rather than silently + // escalating this agent to full privilege. So a delivery failure here + // means this agent's GitHub writes (gh + git push) will fail until + // token delivery is repaired — not a silent privilege escalation. + m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", + "agent", agent.Name, "tier", tier, "error", err) + m.clearCondition(crux.ClassForgeUnreachable, agent.Name) + m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") + } else { + m.appNotInstalledWarned.Delete(agent.Name) + m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) + m.clearCondition(crux.ClassForgeUnreachable, agent.Name) + } + // Additionally issue an opt-in short-lived mint token (no-op when the mint + // is disabled). This is additive and fail-safe — it never blocks launch. + if agent.UID > 0 { + m.issueAgentMintToken(agent.Name, tier, agent.UID) + } +} + +// isForgeTransportErr reports whether a mint error never got an answer from +// the GitHub host: a deadline, a *url.Error (dial/TLS/timeout from the HTTP +// client) or any net.Error. An HTTP status from GitHub (401/404/422) is not +// a transport error and keeps its mint-failed classification. Mirrors the +// dashboard's forge probe (#17427) so both report the same outage. +func isForgeTransportErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var urlErr *url.Error + if errors.As(err, &urlErr) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) +} diff --git a/src/crux/agent/manager_tokens.go b/src/crux/agent/manager_tokens.go index 44cc4933dc..93affc50fc 100644 --- a/src/crux/agent/manager_tokens.go +++ b/src/crux/agent/manager_tokens.go @@ -2,7 +2,6 @@ package agent import ( "context" - "errors" "fmt" "os" "strings" @@ -903,60 +902,6 @@ func (m *Manager) refreshAgentTokens(ctx context.Context) { func truncateStr(s string, maxRunes int) string { return textutil.Head(s, maxRunes) } -// mintAgentTokenUnlocked mints the per-agent tier-scoped GitHub App token -// (WriteAgentToken) and the opt-in short-lived mint token for the agent. It is -// the single mint step every (re)launch path — Start, Resume, Restart, -// RestartWithBootstrap — runs before launchInTmux (#3962: previously only -// Start minted, so an agent relaunched via Resume/Restart never got a token -// until the whole process restarted; its cache file stayed 0 bytes and gh/git -// push failed silently). -// -// MUST be called with m.mu RELEASED. WriteAgentToken makes an outbound GitHub -// API call (through the MITM egress proxy, which the production incident -// showed can hang), and issueAgentMintToken can do the same. Holding m.mu -// across a hang stalls AllStatuses()/the heartbeat for the WHOLE fleet — the -// exact liveness flap Start's phase split was built to avoid. The function -// reads only immutable agent identity (Name, UID) and Config via agentMode, -// and writes no m.mu-guarded AgentProcess field, so running it lock-free is -// race-free. m.appAuth is read without the lock exactly as Start's unlocked -// phase always did: it is injected once at wiring time, before agents start. -func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProcess) { - if m.appAuth == nil || !m.agentTokenCacheEnabled(agent) { - return - } - tier := m.agentTokenTier(agent) - if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { - // Static misconfiguration: every tick would fail identically, so log - // once per agent until a mint succeeds instead of every supervisor tick. - if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { - m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", - "agent", agent.Name, "tier", tier, "error", err) - } - m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") - } else if err != nil { - // Be precise about the blast radius. Since audit H3 the shared-cache - // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no - // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the - // FULL installation token, now owner-only 0600). They FAIL LOUD when - // the per-agent scoped token is absent rather than silently - // escalating this agent to full privilege. So a delivery failure here - // means this agent's GitHub writes (gh + git push) will fail until - // token delivery is repaired — not a silent privilege escalation. - m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", - "agent", agent.Name, "tier", tier, "error", err) - m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") - } else { - m.appNotInstalledWarned.Delete(agent.Name) - m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) - m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) - } - // Additionally issue an opt-in short-lived mint token (no-op when the mint - // is disabled). This is additive and fail-safe — it never blocks launch. - if agent.UID > 0 { - m.issueAgentMintToken(agent.Name, tier, agent.UID) - } -} - // agentTokenCacheEnabled keeps the shared-UID read path opt-in and App-only. func (m *Manager) agentTokenCacheEnabled(a *AgentProcess) bool { return a.UID > 0 || (a.UID == 0 && m.project.SharedUIDReadToken && m.appAuth != nil) diff --git a/src/crux/agent/mint_forge_unreachable_test.go b/src/crux/agent/mint_forge_unreachable_test.go new file mode 100644 index 0000000000..69e8cdd055 --- /dev/null +++ b/src/crux/agent/mint_forge_unreachable_test.go @@ -0,0 +1,93 @@ +package agent + +import ( + "context" + "errors" + "net" + "net/url" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +// failingMinter returns the configured error from WriteAgentToken. +type failingMinter struct{ err error } + +func (f *failingMinter) WriteAgentToken(context.Context, string, string, int) error { return f.err } + +type fakeNetErr struct{} + +func (fakeNetErr) Error() string { return "dial tcp 129.34.104.21:443: connect: connection refused" } +func (fakeNetErr) Timeout() bool { return false } +func (fakeNetErr) Temporary() bool { return false } + +var _ net.Error = fakeNetErr{} + +func mintClassManager(t *testing.T, minter AppTokenMinter) (*Manager, *AgentProcess) { + t.Helper() + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + m := NewManager(map[string]config.AgentConfig{ + "worker": makeAgentConfig("claude", "sonnet"), + }, discardLogger(), ProjectContext{}) + m.mu.Lock() + a := m.agents["worker"] + a.UID = 2101 + m.mu.Unlock() + m.SetAppAuth(minter) + return m, a +} + +// A mint that never reaches the forge is a forge_unreachable condition, not a +// github_app_token_mint_failed one — the App is not implicated by an egress +// outage, and the mint-failed condition is cleared so both are never open. +func TestMintTransportErrorReportsForgeUnreachable(t *testing.T) { + cases := map[string]error{ + "deadline": context.DeadlineExceeded, + "url_error": &url.Error{Op: "Post", URL: "https://github.ibm.com/api/v3/app/installations/45218/access_tokens", Err: context.DeadlineExceeded}, + "net_error": errors.Join(errors.New("creating installation token"), fakeNetErr{}), + } + for name, err := range cases { + t.Run(name, func(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: err}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.active(crux.ClassForgeUnreachable); len(got) != 1 || got[0].Key != "worker" { + t.Fatalf("forge_unreachable active = %+v, want one for worker", got) + } + if got := rec.observations(crux.ClassGitHubAppTokenMintFailed); len(got) != 0 { + t.Fatalf("github_app_token_mint_failed observed on a transport error: %+v", got) + } + if got := rec.clears(crux.ClassGitHubAppTokenMintFailed); len(got) != 1 { + t.Fatalf("mint-failed clears = %d, want 1", len(got)) + } + }) + } +} + +// An answer from the forge that refuses the mint keeps the mint-failed class +// and clears any forge_unreachable left over from an outage. +func TestMintNonTransportErrorKeepsMintFailed(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: errors.New("creating installation token: POST .../access_tokens: 401 Bad credentials")}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.active(crux.ClassGitHubAppTokenMintFailed); len(got) != 1 { + t.Fatalf("github_app_token_mint_failed active = %+v, want one", got) + } + if got := rec.observations(crux.ClassForgeUnreachable); len(got) != 0 { + t.Fatalf("forge_unreachable observed on an HTTP error: %+v", got) + } + if got := rec.clears(crux.ClassForgeUnreachable); len(got) != 1 { + t.Fatalf("forge_unreachable clears = %d, want 1", len(got)) + } +} + +// A successful mint clears every mint-related condition, including the new one. +func TestMintSuccessClearsForgeUnreachable(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: nil}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.clears(crux.ClassForgeUnreachable); len(got) != 1 { + t.Fatalf("forge_unreachable clears = %d, want 1", len(got)) + } +} diff --git a/src/crux/conditions.go b/src/crux/conditions.go index 3ee1942289..0175629224 100644 --- a/src/crux/conditions.go +++ b/src/crux/conditions.go @@ -37,9 +37,16 @@ const ( ClassDataDiskPressure = "data_disk_pressure" ClassGitHubAppTokenMintFailed = "github_app_token_mint_failed" ClassGitHubAppNotInstalled = "github_app_not_installed" - ClassAgentStall = "agent_stall" - ClassAgentCrashLoop = "agent_crash_loop" - ClassCIPollNudge = "ci_poll_nudge" + // ClassForgeUnreachable: a per-agent token mint failed at the transport + // layer (dial timeout, connection refused, context deadline) — the GitHub + // host cannot be reached from the pod. The App key and installation are + // not implicated, so this is kept apart from + // ClassGitHubAppTokenMintFailed, which would otherwise send the operator + // off to re-check App permissions during a network outage. + ClassForgeUnreachable = "forge_unreachable" + ClassAgentStall = "agent_stall" + ClassAgentCrashLoop = "agent_crash_loop" + ClassCIPollNudge = "ci_poll_nudge" // ClassRuntimeUnavailable is raised by the HOST, not the runtime: a // sidecar host whose runtime API stopped answering (hivecommons/hive#14138 // stage 4). The runtime cannot report its own absence. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 8a3f887968..88fcc6b764 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -148,6 +148,7 @@ title is `[selfreport] <class>: <short message>`. | `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | +| `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index 4d1ea1a596..25678d8042 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -98,6 +98,7 @@ var selfreportOpsRemedies = map[string]string{ "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", "data_disk_pressure": "free or grow the data disk on the spoke", "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 6e17ebf4ca..b73565886e 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -168,7 +168,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassGitHubAppAlertsPermission, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassForgeUnreachable, fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 5e56884fd9..62362f055e 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -39,6 +39,7 @@ const ( ClassDataDiskPressure = crux.ClassDataDiskPressure ClassGitHubAppTokenMintFailed = crux.ClassGitHubAppTokenMintFailed ClassGitHubAppNotInstalled = crux.ClassGitHubAppNotInstalled + ClassForgeUnreachable = crux.ClassForgeUnreachable ClassUnexpectedHub5xx = "unexpected_http_5xx_from_hub" ClassUnclassifiedRepeated = "unclassified_error_repeated" // ClassAgentStall, ClassAgentCrashLoop and ClassCIPollNudge replace the @@ -183,6 +184,7 @@ var Classes = map[string]ClassSpec{ ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, + ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index ec5791dc89..8ff311c552 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -433,6 +433,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassUpgradeStalled, message: "standalone upgrade helper did not finish within 10m0s"}, {class: ClassDataDiskPressure, message: "data disk pressure: disk-full: /data is 97% used (threshold 95%)"}, {class: ClassGitHubAppTokenMintFailed, message: "github app token mint failed: POST /app/installations/123/access_tokens: 401"}, + {class: ClassForgeUnreachable, message: "github host unreachable during token mint: creating installation token: context deadline exceeded"}, {class: ClassUnexpectedHub5xx, message: "hub heartbeat returned HTTP 503"}, {class: ClassUnclassifiedRepeated, message: "weird error from somewhere"}, {class: ClassAgentStall, message: "agent stalled after kick: 3 continue nudges sent without the pane moving"}, From 817877e979f545b88ad8a54679427ce7958ff4bc Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 18:25:42 +0000 Subject: [PATCH 102/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-dependabot-permission-selfreport.md | 1 - changelog.d/fixed-signed-reconcile-unsupported.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-dependabot-permission-selfreport.md delete mode 100644 changelog.d/fixed-signed-reconcile-unsupported.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 73d850f252..8d46ba2981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.2) + +### Fixed + +- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. +- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. + ## 2026-11-12 (v5.236.1) ### Changed diff --git a/changelog.d/fixed-dependabot-permission-selfreport.md b/changelog.d/fixed-dependabot-permission-selfreport.md deleted file mode 100644 index 2438213269..0000000000 --- a/changelog.d/fixed-dependabot-permission-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. diff --git a/changelog.d/fixed-signed-reconcile-unsupported.md b/changelog.d/fixed-signed-reconcile-unsupported.md deleted file mode 100644 index ad1001d019..0000000000 --- a/changelog.d/fixed-signed-reconcile-unsupported.md +++ /dev/null @@ -1 +0,0 @@ -- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. From c6c8a0bf85334b677941537504ea3c4d5dbf604c Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 18:57:28 +0000 Subject: [PATCH 103/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.3 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-canceled-adoption-report.md | 1 - changelog.d/fixed-forge-unreachable-mint-class.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-canceled-adoption-report.md delete mode 100644 changelog.d/fixed-forge-unreachable-mint-class.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d46ba2981..d45b2467b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.3) + +### Fixed + +- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. +- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. + ## 2026-11-12 (v5.236.2) ### Fixed diff --git a/changelog.d/fixed-canceled-adoption-report.md b/changelog.d/fixed-canceled-adoption-report.md deleted file mode 100644 index 803255f554..0000000000 --- a/changelog.d/fixed-canceled-adoption-report.md +++ /dev/null @@ -1 +0,0 @@ -- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. diff --git a/changelog.d/fixed-forge-unreachable-mint-class.md b/changelog.d/fixed-forge-unreachable-mint-class.md deleted file mode 100644 index 65050c8d68..0000000000 --- a/changelog.d/fixed-forge-unreachable-mint-class.md +++ /dev/null @@ -1 +0,0 @@ -- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. From 9dcfab4a6911fe9bbfb975589dc1a28bbc2ee06c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:13:05 +0000 Subject: [PATCH 104/223] =?UTF-8?q?=F0=9F=90=9B=20fix(crux):=20prepare=20p?= =?UTF-8?q?luk=20session=20log=20without=20following=20planted=20symlinks?= =?UTF-8?q?=20(#17967)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensurePlukLogFile ran os.Chmod (and, as root, os.Chown) on a path inside /var/run/pluk/logs, a 0770+setgid directory every agent uid can write to. A symlink planted there by an agent at its own predictable session-log name was followed on the next relaunch, widening the link's target to the shared group. Open with O_NOFOLLOW|O_NONBLOCK, require a regular file on the descriptor, and apply owner and mode through the descriptor; a symlink or FIFO found at the path is replaced like an unopenable stale log already was. Fixes #17966 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-17966-pluk-log-nofollow.md | 1 + src/crux/agent/pluk_dirs.go | 64 +++++++++++--- src/crux/agent/pluk_dirs_branches_test.go | 88 +++++++++++++++++++ 3 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 changelog.d/security-17966-pluk-log-nofollow.md diff --git a/changelog.d/security-17966-pluk-log-nofollow.md b/changelog.d/security-17966-pluk-log-nofollow.md new file mode 100644 index 0000000000..338e015b68 --- /dev/null +++ b/changelog.d/security-17966-pluk-log-nofollow.md @@ -0,0 +1 @@ +- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). diff --git a/src/crux/agent/pluk_dirs.go b/src/crux/agent/pluk_dirs.go index a8bd56c0c2..6273fb36ba 100644 --- a/src/crux/agent/pluk_dirs.go +++ b/src/crux/agent/pluk_dirs.go @@ -6,6 +6,9 @@ import ( "io/fs" "os" "path/filepath" + "syscall" + + "github.com/hivecommons/hive/crux/sharedfile" ) // plukRunDir is where pluk's run state and per-session logs live. @@ -45,42 +48,79 @@ func plukSessionLogPath(runDir, session string) string { // ensurePlukLogFile creates (or leaves alone) the session's JSONL log with a // mode peer agents can read, and returns its path. Both the O_CREATE mode and a -// following Chmod are needed: O_CREATE is umask-filtered, and an existing file +// following chmod are needed: O_CREATE is umask-filtered, and an existing file // from an earlier run under a tighter umask has to be widened back to the shared // mode rather than inherited as-is. +// +// The logs dir is group-writable by every agent uid (0770+setgid, no sticky +// bit), so anything found at path may have been planted by an agent: the open +// never follows a symlink, the descriptor is checked to be a regular file, +// and the owner and mode are applied through that descriptor, never by path. +// A path-based chmod/chown here would let one agent widen any dev-owned file +// to the shared group by linking its own session log at it (#17966). func ensurePlukLogFile(runDir, session string) (string, error) { path := plukSessionLogPath(runDir, session) - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) - if errors.Is(err, fs.ErrPermission) { + f, err := openPlukLogNoFollow(path) + if errors.Is(err, fs.ErrPermission) || errors.Is(err, sharedfile.ErrNotRegular) { // A log left behind by another uid (a pane shell's `>>` under a tight // umask, or an earlier run as a different user) cannot be opened or - // chmodded by us, but the shared setgid logs dir is group-writable, so - // the stale file can be replaced. Replacing beats leaving the session's - // log unreadable to peers. + // chmodded by us, and a symlink or other non-regular entry at the path + // is not a log at all. The shared setgid logs dir is group-writable, so + // the entry can be replaced. Replacing beats leaving the session's log + // unreadable to peers, or following a planted link. if rmErr := os.Remove(path); rmErr == nil { - f, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + f, err = openPlukLogNoFollow(path) } } if err != nil { return "", fmt.Errorf("creating %s: %w", path, err) } - if err := f.Close(); err != nil { - return "", fmt.Errorf("closing %s: %w", path, err) - } + defer f.Close() if os.Geteuid() == 0 { - if err := os.Chown(path, DevUID, NodeGID); err != nil { + if err := f.Chown(DevUID, NodeGID); err != nil { return "", fmt.Errorf("chowning %s to %d:%d: %w", path, DevUID, NodeGID, err) } } - if err := os.Chmod(path, plukLogFileMode); err != nil { + if err := f.Chmod(plukLogFileMode); err != nil { return "", fmt.Errorf("chmoding %s to %s: %w", path, plukLogFileMode, err) } + if err := f.Close(); err != nil { + return "", fmt.Errorf("closing %s: %w", path, err) + } return path, nil } +// openPlukLogNoFollow opens (creating if absent) the regular file at path for +// append without following a symlink there. A symlink is reported as +// sharedfile.ErrNotRegular (the kernel answers ELOOP), as is any other +// non-regular file found behind the descriptor. O_NONBLOCK keeps a planted +// FIFO from parking the launch path on open (it fails with ENXIO instead); +// the kernel ignores it for a regular file. +func openPlukLogNoFollow(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, plukLogFileMode) + if err != nil { + // ELOOP: a symlink. ENXIO: a FIFO with no reader (O_NONBLOCK). Neither + // is a log; both are reported as not-regular so the caller replaces them. + if errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.ENXIO) { + return nil, fmt.Errorf("%w: %s (%v)", sharedfile.ErrNotRegular, path, err) + } + return nil, err + } + info, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + if !info.Mode().IsRegular() { + f.Close() + return nil, fmt.Errorf("%w: %s is %s", sharedfile.ErrNotRegular, path, info.Mode().Type()) + } + return f, nil +} + func ensurePlukRunDirs(runDir string) error { for _, name := range []string{"logs", "commands"} { dir := filepath.Join(runDir, name) diff --git a/src/crux/agent/pluk_dirs_branches_test.go b/src/crux/agent/pluk_dirs_branches_test.go index 5cac61004b..0d0affa096 100644 --- a/src/crux/agent/pluk_dirs_branches_test.go +++ b/src/crux/agent/pluk_dirs_branches_test.go @@ -6,7 +6,9 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" + "time" ) // pluk_dirs_branches_test.go covers the pluk_dirs.go branches the setgid-mode @@ -192,3 +194,89 @@ func TestEnsurePlukRunDirsRewidensTightenedDirs(t *testing.T) { } } } + +// TestEnsurePlukLogFileDoesNotFollowPlantedSymlink pins #17966: the logs dir is +// group-writable by every agent uid, so a symlink at the session log path may +// have been planted by an agent. The chmod (and, as root, the chown) must never +// reach the link's target; the link is replaced by a regular log instead. +func TestEnsurePlukLogFileDoesNotFollowPlantedSymlink(t *testing.T) { + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + secret := filepath.Join(t.TempDir(), "proxy-ca-key.pem") + if err := os.WriteFile(secret, []byte("private"), 0o600); err != nil { + t.Fatalf("seed secret: %v", err) + } + path := plukSessionLogPath(runDir, "hive-attacker") + if err := os.Symlink(secret, path); err != nil { + t.Fatalf("plant symlink: %v", err) + } + + got, err := ensurePlukLogFile(runDir, "hive-attacker") + if err != nil { + t.Fatalf("ensure pluk log file over a planted symlink: %v", err) + } + if got != path { + t.Errorf("returned path = %q, want %q", got, path) + } + + info, err := os.Stat(secret) + if err != nil { + t.Fatalf("stat secret: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Fatalf("symlink target mode = %04o, want 0600 untouched (chmod followed the planted link)", perm) + } + data, err := os.ReadFile(secret) + if err != nil || string(data) != "private" { + t.Fatalf("symlink target changed: %q, %v", data, err) + } + + li, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if !li.Mode().IsRegular() { + t.Fatalf("log path is %s, want a regular file replacing the planted link", li.Mode().Type()) + } + if perm := li.Mode().Perm(); perm != 0o660 { + t.Errorf("replaced log mode = %04o, want 0660", perm) + } +} + +// TestEnsurePlukLogFileReplacesPlantedFIFO covers the other non-regular entry +// an agent can create in the shared dir: a FIFO with no reader must neither +// block the launch path on open nor be kept; it is replaced like a symlink. +func TestEnsurePlukLogFileReplacesPlantedFIFO(t *testing.T) { + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + path := plukSessionLogPath(runDir, "hive-fifo") + if err := syscall.Mkfifo(path, 0o660); err != nil { + t.Fatalf("plant fifo: %v", err) + } + + done := make(chan error, 1) + go func() { + _, err := ensurePlukLogFile(runDir, "hive-fifo") + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Fatalf("ensure pluk log file over a planted fifo: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("ensure pluk log file blocked on a planted fifo") + } + + li, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if !li.Mode().IsRegular() { + t.Fatalf("log path is %s, want a regular file replacing the fifo", li.Mode().Type()) + } +} From 2b94c9f3a45fd331ab8c918d45d3285dbe5be716 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 19:43:43 +0000 Subject: [PATCH 105/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.4 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/security-17966-pluk-log-nofollow.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/security-17966-pluk-log-nofollow.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d45b2467b5..f7f0019c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.4) + +### Security + +- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). + ## 2026-11-12 (v5.236.3) ### Fixed diff --git a/changelog.d/security-17966-pluk-log-nofollow.md b/changelog.d/security-17966-pluk-log-nofollow.md deleted file mode 100644 index 338e015b68..0000000000 --- a/changelog.d/security-17966-pluk-log-nofollow.md +++ /dev/null @@ -1 +0,0 @@ -- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). From 01de5d33344e9ea8a097532629397206bfa457f3 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:47:15 +0100 Subject: [PATCH 106/223] =?UTF-8?q?=F0=9F=90=9B=20fix(fleetreport):=20reco?= =?UTF-8?q?gnize=20structured=20context=20cancellation=20(#17937)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(fleetreport): recognize structured context cancellation Apply the existing cancellation exclusion to structured error attributes while keeping local logs and other errors visible. Hive-Run: hivecommons/hive#17935 Hive-Plan: structured-cancellation-selfreport Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * fix(fleetreport): recognize wrapped cancellations Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ixed-structured-cancellation-selfreport.md | 1 + src/docs/fleet-report.md | 6 ++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 35 +++++++++++-------- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 28 ++++++++++++++- 4 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-structured-cancellation-selfreport.md diff --git a/changelog.d/fixed-structured-cancellation-selfreport.md b/changelog.d/fixed-structured-cancellation-selfreport.md new file mode 100644 index 0000000000..5d9018befc --- /dev/null +++ b/changelog.d/fixed-structured-cancellation-selfreport.md @@ -0,0 +1 @@ +- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 88fcc6b764..aab1ac0c3f 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -171,6 +171,12 @@ title is `[selfreport] <class>: <short message>`. | `models_endpoint_empty` | the dashboard's inference model discovery (`/api/inference/{backend}/models`) found no models from any configured endpoint (`no models found from any endpoint`, `pkg/dashboard/api_models.go`) and fell back to the static aliases, flagged unverified in the UI. The source logs it at WARN once per backend per hour (DEBUG on repeats, re-armed when discovery succeeds), so this fires only when the condition flaps ≥5 times in 10 min (gateway key rejected, endpoint down). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | +The cancellation exclusion includes structured `error` attributes, including +logger-bound attributes: typed errors wrapping `context.Canceled`, exact +`context canceled` strings, and strings ending in `: context canceled`. +Warnings remain in local logs; deadlines and other errors remain reportable, +including when another `error` attribute accompanies a cancellation. + The GitHub rate-limit exclusion also covers the local REST reserve guard: when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 1550bafe9b..e35e8781fd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -115,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if canceledAdoptionRead(r) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -136,22 +136,29 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } -// A canceled sweep leaves the existing adoption intact and retries later. -// Other failures reading push access still warrant a repeated-error report. -func canceledAdoptionRead(r slog.Record) bool { - if strings.TrimSpace(r.Message) != "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" { - return false - } - canceled := false - r.Attrs(func(a slog.Attr) bool { +// canceledError applies the existing cancellation exclusion to structured +// errors too. An accompanying different error must remain reportable. +func canceledError(r slog.Record, bound []slog.Attr) bool { + found, other := false, false + check := func(a slog.Attr) bool { if a.Key == "error" { - err, ok := a.Value.Any().(error) - canceled = (ok && errors.Is(err, context.Canceled)) || a.Value.String() == context.Canceled.Error() - return !canceled + value := a.Value.Resolve() + err, ok := value.Any().(error) + if ok && errors.Is(err, context.Canceled) { + found = true + } else if text := value.String(); text == context.Canceled.Error() || strings.HasSuffix(text, ": "+context.Canceled.Error()) { + found = true + } else { + other = true + } } return true - }) - return canceled + } + for _, a := range bound { + check(a) + } + r.Attrs(check) + return found && !other } // expectedKickHold reports whether a "failed to send kick" record carries the diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index ce7c2d09ab..838050edbd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "log/slog" "net/url" "strings" @@ -77,7 +78,32 @@ func TestCanceledAdoptionReadStaysInLog(t *testing.T) { logger.Warn(msg, "error", errors.New("permission denied")) logger.Warn(msg, "error", context.DeadlineExceeded) logger.Warn("another operation failed", "error", context.Canceled) - if len(*seen) != 3 { + if len(*seen) != 2 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + +func TestStructuredCancellationDoesNotCreateFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels reconcile failed" + for range 10 { + logger.Warn(msg, "error", context.Canceled, "items", 6, "writes", 0) + } + logger.With("error", context.Canceled).Warn(msg) + logger.Warn("stage labels: check digest failed", "error", fmt.Errorf("GET checks: %w", context.Canceled)) + logger.Warn("stage labels: check digest failed", "error", "GET checks: context canceled") + logger.With("error", fmt.Errorf("listing PRs: %w", context.Canceled)).Warn(msg, "error", context.Canceled) + logger.Warn(msg, "error", &url.Error{Op: "GET", URL: "https://api.github.com", Err: context.Canceled}) + if len(*seen) != 0 || strings.Count(out.String(), "\n") != 15 { + t.Fatalf("observed=%+v logs=%s", *seen, out.String()) + } + logger.Warn(msg, "error", context.DeadlineExceeded) + logger.Warn(msg, "error", "permission denied") + logger.With("error", context.Canceled).Warn(msg, "error", "permission denied") + logger.With("error", fmt.Errorf("listing PRs: %w", context.Canceled)).Warn(msg, "error", "permission denied") + logger.Warn(msg, "error", "context canceled: permission denied") + logger.Warn(msg, "error", fmt.Errorf("request: %w", context.DeadlineExceeded)) + if len(*seen) != 6 { t.Fatalf("other failures must remain reportable: %+v", *seen) } } From dc9ef4a010ece35d8c20cd2d9faf5401273d7829 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:56:27 +0000 Subject: [PATCH 107/223] =?UTF-8?q?=E2=9C=A8=20crux=20proxy:=20add=20OTel?= =?UTF-8?q?=20metrics=20and=20span=20to=20inference=20translator=20(#17974?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inference hop was only observable via slog. Follow the serve/metrics.go pattern with bounded attributes (status_class, outcome enum) so nothing exports unless OTEL_* is configured. Refs hivecommons/crux#153 (re-homed from hivecommons/crux#155: crux Go code is synced from src/crux here) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-crux-proxy-inference-otel.md | 1 + src/crux/proxy/github_proxy_inference.go | 4 +- src/crux/proxy/metrics.go | 164 ++++++++++++++++++ src/crux/proxy/metrics_test.go | 89 ++++++++++ 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added-crux-proxy-inference-otel.md create mode 100644 src/crux/proxy/metrics.go create mode 100644 src/crux/proxy/metrics_test.go diff --git a/changelog.d/added-crux-proxy-inference-otel.md b/changelog.d/added-crux-proxy-inference-otel.md new file mode 100644 index 0000000000..2ed23a192b --- /dev/null +++ b/changelog.d/added-crux-proxy-inference-otel.md @@ -0,0 +1 @@ +- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 4d1e6b4d35..154abf41f1 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -397,6 +397,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { if !ok { p.logger.Warn("inference request from unidentified caller refused", p.translatorRefusalAttrs(r)...) + setTranslatorOutcome(r.Context(), outcomeUnidentifiedCaller) http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) return } @@ -412,6 +413,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { route := p.inference.Get(agentName) if route == nil { p.logger.Warn("inference no route", "agent", agentName, "reason", p.inference.MissReason(agentName)) + setTranslatorOutcome(r.Context(), outcomeNoRoute) http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) return } @@ -585,7 +587,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { _, _ = w.Write(translated) }) - return mux + return instrumentTranslator(mux) } // StartInferenceTranslator runs the Anthropic-compatible inference translator diff --git a/src/crux/proxy/metrics.go b/src/crux/proxy/metrics.go new file mode 100644 index 0000000000..bc7c4e0311 --- /dev/null +++ b/src/crux/proxy/metrics.go @@ -0,0 +1,164 @@ +package proxy + +import ( + "context" + "net/http" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +// Inference translator instruments (hivecommons/crux#153). Like serve's they +// sit on the global providers, which are OTel's delegating no-ops until a host +// registers an SDK provider, so recording costs nothing by default. +// +// Attributes are bounded to the status class and a fixed outcome enum. Never an +// agent name, path, model name, token or body. +const ( + instrumentationName = "github.com/hivecommons/hive/crux/proxy" + + translatorDurationMetric = "crux.proxy.inference.duration" + translatorCountMetric = "crux.proxy.inference.requests" +) + +// Outcome of one translator request: a fixed enum, so the attribute stays bounded. +const ( + outcomeOK = "ok" + outcomeUnidentifiedCaller = "unidentified_caller" + outcomeNoRoute = "no_route" + outcomeUpstreamError = "upstream_error" +) + +var ( + attrStatusClass = attribute.Key("status_class") + attrOutcome = attribute.Key("outcome") +) + +type translatorMetrics struct { + duration metric.Float64Histogram + requests metric.Int64Counter +} + +func newTranslatorMetrics(meter metric.Meter) translatorMetrics { + // Instrument constructors only fail on an invalid name or unit; both are + // constants here, and the returned instrument is a usable no-op either way. + duration, _ := meter.Float64Histogram(translatorDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time the inference translator took to answer a request.")) + requests, _ := meter.Int64Counter(translatorCountMetric, + metric.WithDescription("Inference translator requests, by status class and outcome.")) + return translatorMetrics{duration: duration, requests: requests} +} + +var globalTranslatorMetrics = sync.OnceValue(func() translatorMetrics { + return newTranslatorMetrics(otel.Meter(instrumentationName)) +}) + +func (m translatorMetrics) record(ctx context.Context, d time.Duration, attrs []attribute.KeyValue) { + opt := metric.WithAttributes(attrs...) + m.duration.Record(ctx, d.Seconds(), opt) + m.requests.Add(ctx, 1, opt) +} + +type outcomeKey struct{} + +// outcomeHolder lets the handler name an outcome the status alone cannot tell. +type outcomeHolder struct{ outcome string } + +func setTranslatorOutcome(ctx context.Context, outcome string) { + if h, ok := ctx.Value(outcomeKey{}).(*outcomeHolder); ok { + h.outcome = outcome + } +} + +type translatorStatusRecorder struct { + http.ResponseWriter + status int +} + +func (w *translatorStatusRecorder) WriteHeader(status int) { + if w.status == 0 { + w.status = status + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *translatorStatusRecorder) Write(b []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.ResponseWriter.Write(b) +} + +func (w *translatorStatusRecorder) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +// translatorFlushRecorder offers Flush only when the wrapped writer does; the +// SSE path type-asserts http.Flusher. +type translatorFlushRecorder struct { + *translatorStatusRecorder + http.Flusher +} + +func translatorStatusClass(status int) string { + switch { + case status >= 500: + return "5xx" + case status >= 400: + return "4xx" + case status >= 300: + return "3xx" + case status >= 200: + return "2xx" + default: + return "1xx" + } +} + +// instrumentTranslator records one request's server span, duration and count +// around next. +func instrumentTranslator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + holder := &outcomeHolder{} + ctx, span := otel.Tracer(instrumentationName).Start( + context.WithValue(r.Context(), outcomeKey{}, holder), + "inference_translator", trace.WithSpanKind(trace.SpanKindServer)) + rec := &translatorStatusRecorder{ResponseWriter: w} + var out http.ResponseWriter = rec + if f, ok := w.(http.Flusher); ok { + out = translatorFlushRecorder{rec, f} + } + returned := false + defer func() { + status := rec.status + if status == 0 { + status = http.StatusOK + } + if !returned { + status = http.StatusInternalServerError + span.SetStatus(codes.Error, "handler panic") + } + outcome := holder.outcome + if outcome == "" { + outcome = outcomeOK + if status >= 400 { + outcome = outcomeUpstreamError + } + } + attrs := []attribute.KeyValue{ + attrStatusClass.String(translatorStatusClass(status)), + attrOutcome.String(outcome), + } + span.SetAttributes(attrs...) + span.End() + globalTranslatorMetrics().record(ctx, time.Since(start), attrs) + }() + next.ServeHTTP(out, r.WithContext(ctx)) + returned = true + }) +} diff --git a/src/crux/proxy/metrics_test.go b/src/crux/proxy/metrics_test.go new file mode 100644 index 0000000000..0fcb9b8df4 --- /dev/null +++ b/src/crux/proxy/metrics_test.go @@ -0,0 +1,89 @@ +package proxy + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/hivecommons/hive/crux/agent" + + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestTranslatorMetricsBoundedAttributes(t *testing.T) { + fakeProcNetTCP(t) + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + m := newTranslatorMetrics(mp.Meter(instrumentationName)) + prev := globalTranslatorMetrics + globalTranslatorMetrics = func() translatorMetrics { return m } + t.Cleanup(func() { + globalTranslatorMetrics = prev + _ = mp.Shutdown(context.Background()) + }) + + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, Agents: map[string]int{"quality": 2002}} + rec := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(rec, translatorReq("127.0.0.1:43210", "quality")) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", rec.Code) + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + var counts, durations int + for _, sm := range rm.ScopeMetrics { + for _, mt := range sm.Metrics { + got := map[string]string{} + collect := func(len int, kv func(string) (string, bool)) { + if len != 2 { + t.Errorf("%s: %d attributes, want exactly status_class and outcome", mt.Name, len) + } + for _, k := range []string{"status_class", "outcome", "agent", "path", "model"} { + if v, ok := kv(k); ok { + got[k] = v + } + } + } + switch data := mt.Data.(type) { + case metricdata.Sum[int64]: + counts++ + for _, dp := range data.DataPoints { + collect(dp.Attributes.Len(), func(k string) (string, bool) { + v, ok := dp.Attributes.Value(attrKey(k)) + return v.AsString(), ok + }) + } + case metricdata.Histogram[float64]: + durations++ + for _, dp := range data.DataPoints { + collect(dp.Attributes.Len(), func(k string) (string, bool) { + v, ok := dp.Attributes.Value(attrKey(k)) + return v.AsString(), ok + }) + } + } + if got["status_class"] != "4xx" || got["outcome"] != outcomeUnidentifiedCaller { + t.Errorf("%s attributes = %v, want 4xx/unidentified_caller", mt.Name, got) + } + for _, bad := range []string{"agent", "path", "model"} { + if _, ok := got[bad]; ok { + t.Errorf("%s carries forbidden attribute %q", mt.Name, bad) + } + } + } + } + if counts != 1 || durations != 1 { + t.Fatalf("counters = %d, histograms = %d, want 1 each", counts, durations) + } +} + +func attrKey(k string) attribute.Key { return attribute.Key(k) } From a3d41aaf19c61b4fc5f6208a67d1b4c99e3bd3f3 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:17:05 +0000 Subject: [PATCH 108/223] =?UTF-8?q?=F0=9F=90=9B=20fix(crux/proxy):=20make?= =?UTF-8?q?=20CONNECT-refusal=20test=20hermetic=20(#17981)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestDialUpstreamTLSSurfacesProxyRefusal relied on the test CONNECT proxy failing to resolve ghe.corp.invalid within 3s; on CI the resolver hung and the client timed out instead of seeing the 502. The test proxy now refuses on demand so the test does not depend on DNS. Closes #17979 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/proxy/upstream_chain_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/crux/proxy/upstream_chain_test.go b/src/crux/proxy/upstream_chain_test.go index 4c902d3483..9c68b4b5bf 100644 --- a/src/crux/proxy/upstream_chain_test.go +++ b/src/crux/proxy/upstream_chain_test.go @@ -23,6 +23,7 @@ type testConnectProxy struct { ln net.Listener targets atomic.Int32 last atomic.Value + refuse atomic.Bool } func startTestConnectProxy(t *testing.T) *testConnectProxy { @@ -55,6 +56,10 @@ func (cp *testConnectProxy) serve(c net.Conn) { } cp.targets.Add(1) cp.last.Store(req.Host) + if cp.refuse.Load() { + _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } up, err := net.DialTimeout("tcp", req.Host, 5*time.Second) if err != nil { _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") @@ -240,7 +245,8 @@ func TestDialUpstreamRawChainsAndGuardsPrivateLiterals(t *testing.T) { func TestDialUpstreamTLSSurfacesProxyRefusal(t *testing.T) { cp := startTestConnectProxy(t) p := newChainedTestProxy(t, cp, []string{"ghe.corp.invalid"}) - // The test proxy dials the target itself; an unresolvable name yields 502. + // The test proxy is told to refuse so the test does not depend on DNS. + cp.refuse.Store(true) _, err := p.dialUpstreamTLS("ghe.corp.invalid:443", "ghe.corp.invalid", 3*time.Second) if err == nil || !strings.Contains(err.Error(), "refused") { t.Fatalf("err = %v, want CONNECT refusal", err) From 2a7495999ea8947393c8bfd993589f18d3f3bb09 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 20:21:57 +0000 Subject: [PATCH 109/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.5 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-structured-cancellation-selfreport.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-structured-cancellation-selfreport.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f7f0019c95..5d9d84ce6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.5) + +### Fixed + +- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). + ## 2026-11-12 (v5.236.4) ### Security diff --git a/changelog.d/fixed-structured-cancellation-selfreport.md b/changelog.d/fixed-structured-cancellation-selfreport.md deleted file mode 100644 index 5d9018befc..0000000000 --- a/changelog.d/fixed-structured-cancellation-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). From 9a10ac369f27a14dcba91a443823740f74473703 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 20:53:12 +0000 Subject: [PATCH 110/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.237.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.237.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-crux-proxy-inference-otel.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-crux-proxy-inference-otel.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d9d84ce6d..b12cfc3707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.237.0) + +### Added + +- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). + ## 2026-11-12 (v5.236.5) ### Fixed diff --git a/changelog.d/added-crux-proxy-inference-otel.md b/changelog.d/added-crux-proxy-inference-otel.md deleted file mode 100644 index 2ed23a192b..0000000000 --- a/changelog.d/added-crux-proxy-inference-otel.md +++ /dev/null @@ -1 +0,0 @@ -- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). From 8fa405fd4975616840425e90d67a880452712e36 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:54:02 +0000 Subject: [PATCH 111/223] =?UTF-8?q?=F0=9F=90=9B=20fix(fleetreport):=20name?= =?UTF-8?q?=20the=20prow/lgtm=20statuses-permission=20403=20as=20its=20own?= =?UTF-8?q?=20selfreport=20class=20(#17980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(fleetreport): name the prow/lgtm statuses-permission 403 as its own selfreport class Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📝 fix(fleetreport): document the statuses-permission class and gate it on the 403 Review follow-up for #17980: - fleet-report.md: add the github_app_statuses_permission_missing row beside its sibling github_app_alerts_permission_missing (component, log-fed gate, Commit statuses read & write remedy, github-app-setup.md link). - bindProwLgtm: emit the "lacks the statuses: write permission" wording only when the CreateStatus error is a 403 *gh.ErrorResponse; 5xx, 422 and transport failures log a neutral line, so a transient outage is no longer filed as an installation-permission fault with a grant-permission remedy. - fleetreport keyword now matches the 403-only line; tests pin both the positive match and that the neutral line stays out of the class. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * docs: re-anchor RankActionableIssues citation after client.go shift Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ed-selfreport-statuses-permission-class.md | 1 + src/docs/fleet-report.md | 1 + src/docs/scale-envelope.md | 2 +- src/pkg/delegatedecision/issue_judgment.go | 35 ++-- .../delegatedecision/issue_judgment_test.go | 2 +- src/pkg/fleetreport/runtime.go | 166 +++++++++--------- src/pkg/fleetreport/runtime_test.go | 111 ++++++------ src/pkg/github/automerge_queue_prow_test.go | 38 ++++ src/pkg/github/client.go | 27 ++- 9 files changed, 228 insertions(+), 155 deletions(-) create mode 100644 changelog.d/fixed-selfreport-statuses-permission-class.md diff --git a/changelog.d/fixed-selfreport-statuses-permission-class.md b/changelog.d/fixed-selfreport-statuses-permission-class.md new file mode 100644 index 0000000000..84c964db2a --- /dev/null +++ b/changelog.d/fixed-selfreport-statuses-permission-class.md @@ -0,0 +1 @@ +- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index aab1ac0c3f..bd63e9662b 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -146,6 +146,7 @@ title is `[selfreport] <class>: <short message>`. | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | | `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | +| `github_app_statuses_permission_missing` | GitHub answered the queue action's `prow/lgtm` commit-status write with 403 (`could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission`, `pkg/github/client.go`): the App installation lacks Commit statuses read & write, so Prow may strip the `lgtm` label and the auto-merge queue may not hold. The App owner must grant Commit statuses read & write and re-accept it on the installation. Only the 403 carries this wording — any other status failure (5xx, 422, transport) logs the neutral `could not set the prow/lgtm commit status` and is not this class. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index ea6cfeadf6..af8d9a3c73 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5263-5273`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5286-5296`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index 25678d8042..16718210f7 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -94,23 +94,24 @@ func ParseSelfReport(title, body string) (SelfReport, bool) { // constants (pinned by TestSelfreportOpsClassesMatchFleetReport) without // importing that package's dependency tree into this leaf. var selfreportOpsRemedies = map[string]string{ - "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", - "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", - "data_disk_pressure": "free or grow the data disk on the spoke", - "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", - "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", - "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", - "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", - "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", - "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", - "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", - "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", - "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", - "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", - "inference_upstream_error": "check the inference endpoint the spoke's provider points at", - "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", - "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", - "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", + "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", + "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", + "data_disk_pressure": "free or grow the data disk on the spoke", + "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", + "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", + "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", + "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", + "github_app_statuses_permission_missing": "grant Commit statuses read & write in the GitHub App settings and re-accept the permission on its installation", + "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", + "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", + "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", + "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", + "inference_upstream_error": "check the inference endpoint the spoke's provider points at", + "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", + "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", + "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", } // selfreportOpsClass reports whether a self-report class is an operator's to diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index b73565886e..603bafd44f 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -168,7 +168,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassForgeUnreachable, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassGitHubAppStatusesPermission, fleetreport.ClassForgeUnreachable, fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 62362f055e..90e653690b 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -69,37 +69,38 @@ const ( // Log-fed classes: faults that have no dedicated hook site and are recognised // from their WARN+ log line by the slog feed (see LogFedClass). const ( - ClassTmuxSessionCheckFailed = "tmux_session_check_failed" - ClassAgentPaneTimeout = "agent_pane_timeout" - ClassProxyTLSHandshakeFailed = "proxy_tls_handshake_failed" - ClassAgentDirSetupFailed = "agent_dir_setup_failed" - ClassReviewEvidenceWriteFail = "review_evidence_write_failed" - ClassFeedbackIssueRefreshFailed = "feedback_issue_refresh_failed" - ClassBeadFactIngestFailed = "bead_fact_ingest_failed" - ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" - ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" - ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" - ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" - ClassProxyConnectDialFailed = "proxy_connect_dial_failed" - ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" - ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" - ClassInferenceUpstreamError = "inference_upstream_error" - ClassInferenceCallerUnknown = "inference_caller_unidentified" - ClassInferenceNoRoute = "inference_no_route" - ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" - ClassKickFreshnessWithheld = "kick_freshness_withheld" - ClassModelsEndpointEmpty = "models_endpoint_empty" - ClassAgentLoginRequired = "agent_login_required" - ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" - ClassIOScanClassifierUnavail = "ioscan_classifier_unavailable" - ClassGitHubFetchTransient = "github_fetch_transient" - ClassLabelProvisioningFailed = "label_provisioning_failed" - ClassAgentCLIRestartLoop = "agent_cli_restart_loop" - ClassCopilotCLIHung = "copilot_cli_hung" - ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" - ClassTranscriptScanSlow = "transcript_scan_slow" - ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" - ClassIssueClaimEffectFailed = "issue_claim_effect_failed" + ClassTmuxSessionCheckFailed = "tmux_session_check_failed" + ClassAgentPaneTimeout = "agent_pane_timeout" + ClassProxyTLSHandshakeFailed = "proxy_tls_handshake_failed" + ClassAgentDirSetupFailed = "agent_dir_setup_failed" + ClassReviewEvidenceWriteFail = "review_evidence_write_failed" + ClassFeedbackIssueRefreshFailed = "feedback_issue_refresh_failed" + ClassBeadFactIngestFailed = "bead_fact_ingest_failed" + ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" + ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" + ClassGitHubAppStatusesPermission = "github_app_statuses_permission_missing" + ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" + ClassProxyConnectDialFailed = "proxy_connect_dial_failed" + ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" + ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" + ClassInferenceUpstreamError = "inference_upstream_error" + ClassInferenceCallerUnknown = "inference_caller_unidentified" + ClassInferenceNoRoute = "inference_no_route" + ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" + ClassKickFreshnessWithheld = "kick_freshness_withheld" + ClassModelsEndpointEmpty = "models_endpoint_empty" + ClassAgentLoginRequired = "agent_login_required" + ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" + ClassIOScanClassifierUnavail = "ioscan_classifier_unavailable" + ClassGitHubFetchTransient = "github_fetch_transient" + ClassLabelProvisioningFailed = "label_provisioning_failed" + ClassAgentCLIRestartLoop = "agent_cli_restart_loop" + ClassCopilotCLIHung = "copilot_cli_hung" + ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" + ClassTranscriptScanSlow = "transcript_scan_slow" + ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" + ClassIssueClaimEffectFailed = "issue_claim_effect_failed" // A restart or normal stop can invalidate a pending bootstrap without // indicating a restart storm; only repeated interruptions are reportable. ClassStartupKickInterrupted = "startup_kick_interrupted" @@ -174,57 +175,58 @@ type ClassSpec struct { // Classes is the runtime error allow-list. var Classes = map[string]ClassSpec{ - ClassRestartStorm: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"restart storm", "restart-storm"}}, - ClassKickFailedRepeatedly: {Component: "agent-runtime", Severity: "medium", MinCount: kickFailureStreak, Keywords: []string{"kick failed", "undeliverable kick", "kicks undeliverable", "re-kick backed off"}}, - ClassPanicRecovered: {Component: "dashboard", Severity: "high", MinCount: 1, Keywords: []string{"panic recovered", "recovered panic"}}, - ClassModelDiscoveryDegraded: {Component: "inference-gateway", Severity: "medium", MinCount: 1, MinDuration: degradedMinDuration, Keywords: []string{"model discovery degraded", "model discovery failed"}}, - ClassInferenceAuthFailed: {Component: "backend-auth", Severity: "high", MinCount: 1, Keywords: []string{"litellm 401", "inference 401", "inference 403", "authentication failure"}}, - ClassAdvisoryPostFailed: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"advisory post failed", "advisory digest"}}, - ClassUpgradeStalled: {Component: "agent-lifecycle", Severity: "high", MinCount: 1, Keywords: []string{"upgrade stalled", "upgrade stall"}}, - ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, - ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, - ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, - ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, - ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, - ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, - ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, - ClassAgentCrashLoop: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crash loop", "crash-loop", "crash looping"}}, - ClassCIPollNudge: {Component: "agent-runtime", Severity: "low", MinCount: 1, Window: ciPollNudgeWindow, Keywords: []string{"ci-poll nudge", "polling ci", "stop-polling nudge"}}, - ClassIssueRequestStuck: {Component: "github-relay", Severity: "medium", MinCount: 1, Keywords: []string{"issue-request stuck", "issue request stuck", "issue-request watcher", "relay request stuck"}}, - ClassCruxRuntimeUnavailable: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crux runtime unavailable", "crux server unreachable"}}, - ClassEffectLedgerOpStuck: {Component: "mutation-ledger", Severity: "medium", MinCount: 1, Keywords: []string{"effect ledger stuck", "ledger op stuck", "needs reconciliation", "mutation journal unknown"}}, - ClassTmuxSessionCheckFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"tmux has-session failed"}}, - ClassStartupKickInterrupted: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"startup kick dropped: agent restarted or stopped while waiting"}}, - ClassAgentPaneTimeout: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"prompt timeout — dumping pane", "pane content at timeout", "startup kick dropped: cli never became ready", "startup kick dropped: cli never reached input prompt"}}, - ClassProxyTLSHandshakeFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy client tls handshake failed", "transparent proxy tls handshake failed", "copilot sniff: client tls handshake failed"}}, - ClassAgentDirSetupFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to create per-agent xdg dir", "failed to restore per-agent dir mode", "failed to restore interactive home mode", "failed to tighten per-agent dir mode", "failed to tighten interactive home mode", "failed to restore inference home mode", "failed to tighten inference home mode"}}, - ClassReviewEvidenceWriteFail: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"review evidence: could not write bundle"}}, - ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, - ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, - ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, - ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, - ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, - ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, - ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, - ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, - ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, - ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, - ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, - ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, - ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, - ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, - ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, - ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, - ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, - ClassIOScanClassifierUnavail: {Component: "inference-gateway", Severity: "medium", MinCount: 1, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"ioscan classifier failed closed", "ioscan classifier failed open", "ioscan fail-closed blocked kick", "ioscan fail-closed blocked held-pr coordination kick"}}, - ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, - ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, - ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, - ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, - ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, - ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, - ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, - ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, + ClassRestartStorm: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"restart storm", "restart-storm"}}, + ClassKickFailedRepeatedly: {Component: "agent-runtime", Severity: "medium", MinCount: kickFailureStreak, Keywords: []string{"kick failed", "undeliverable kick", "kicks undeliverable", "re-kick backed off"}}, + ClassPanicRecovered: {Component: "dashboard", Severity: "high", MinCount: 1, Keywords: []string{"panic recovered", "recovered panic"}}, + ClassModelDiscoveryDegraded: {Component: "inference-gateway", Severity: "medium", MinCount: 1, MinDuration: degradedMinDuration, Keywords: []string{"model discovery degraded", "model discovery failed"}}, + ClassInferenceAuthFailed: {Component: "backend-auth", Severity: "high", MinCount: 1, Keywords: []string{"litellm 401", "inference 401", "inference 403", "authentication failure"}}, + ClassAdvisoryPostFailed: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"advisory post failed", "advisory digest"}}, + ClassUpgradeStalled: {Component: "agent-lifecycle", Severity: "high", MinCount: 1, Keywords: []string{"upgrade stalled", "upgrade stall"}}, + ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, + ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, + ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, + ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, + ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, + ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, + ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, + ClassAgentCrashLoop: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crash loop", "crash-loop", "crash looping"}}, + ClassCIPollNudge: {Component: "agent-runtime", Severity: "low", MinCount: 1, Window: ciPollNudgeWindow, Keywords: []string{"ci-poll nudge", "polling ci", "stop-polling nudge"}}, + ClassIssueRequestStuck: {Component: "github-relay", Severity: "medium", MinCount: 1, Keywords: []string{"issue-request stuck", "issue request stuck", "issue-request watcher", "relay request stuck"}}, + ClassCruxRuntimeUnavailable: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crux runtime unavailable", "crux server unreachable"}}, + ClassEffectLedgerOpStuck: {Component: "mutation-ledger", Severity: "medium", MinCount: 1, Keywords: []string{"effect ledger stuck", "ledger op stuck", "needs reconciliation", "mutation journal unknown"}}, + ClassTmuxSessionCheckFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"tmux has-session failed"}}, + ClassStartupKickInterrupted: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"startup kick dropped: agent restarted or stopped while waiting"}}, + ClassAgentPaneTimeout: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"prompt timeout — dumping pane", "pane content at timeout", "startup kick dropped: cli never became ready", "startup kick dropped: cli never reached input prompt"}}, + ClassProxyTLSHandshakeFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy client tls handshake failed", "transparent proxy tls handshake failed", "copilot sniff: client tls handshake failed"}}, + ClassAgentDirSetupFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to create per-agent xdg dir", "failed to restore per-agent dir mode", "failed to restore interactive home mode", "failed to tighten per-agent dir mode", "failed to tighten interactive home mode", "failed to restore inference home mode", "failed to tighten inference home mode"}}, + ClassReviewEvidenceWriteFail: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"review evidence: could not write bundle"}}, + ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, + ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, + ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, + ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, + ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, + ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, + ClassGitHubAppStatusesPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"could not set the prow/lgtm commit status: the github app lacks the statuses: write permission"}}, + ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, + ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, + ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, + ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, + ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, + ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, + ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, + ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, + ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, + ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, + ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, + ClassIOScanClassifierUnavail: {Component: "inference-gateway", Severity: "medium", MinCount: 1, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"ioscan classifier failed closed", "ioscan classifier failed open", "ioscan fail-closed blocked kick", "ioscan fail-closed blocked held-pr coordination kick"}}, + ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, + ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, + ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, + ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, + ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, + ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, + ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, + ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, } // LogFedClass returns the LogFed class whose Keyword msg contains, or "". diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 8ff311c552..7c7a024afe 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -455,6 +455,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, {class: ClassGitHubAppAlertsPermission, message: "scoped token mint with vulnerability_alerts:read failed; retrying without it"}, + {class: ClassGitHubAppStatusesPermission, message: "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission"}, {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, @@ -1200,58 +1201,59 @@ func TestUnclassifiedRepeatedHonoursWindowAndIdleClear(t *testing.T) { func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { fed := map[string]string{ "startup kick dropped: agent restarted or stopped while waiting": ClassStartupKickInterrupted, - "tmux has-session failed": ClassTmuxSessionCheckFailed, - "failed to send kick": ClassAgentLifecycleRepeated, - "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, - "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, - "agent tmux session missing": ClassAgentLifecycleRepeated, - "watchdog: would restart dead agent (observe mode; no action taken)": ClassAgentLifecycleRepeated, - "pane content at timeout": ClassAgentPaneTimeout, - "prompt timeout — dumping pane": ClassAgentPaneTimeout, - "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, - "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, - "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, - "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, - "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, - "copilot hung with no CLI prompt, running diagnostic": ClassCopilotCLIHung, - "diagnostic: timed out waiting for copilot error output, relaunching agent": ClassCopilotCLIHung, - "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, - "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, - "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, - "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, - "no models found from any endpoint": ClassModelsEndpointEmpty, - "login required detected": ClassAgentLoginRequired, - "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, - "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, - "watchdog: agent credential failure": ClassAgentLoginRequired, - "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, - "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, - "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, - "failed to restore interactive home mode": ClassAgentDirSetupFailed, - "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, - "failed to tighten interactive home mode": ClassAgentDirSetupFailed, - "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, - "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, - "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, - "failed to restore inference home mode": ClassAgentDirSetupFailed, - "failed to tighten inference home mode": ClassAgentDirSetupFailed, - "failed to ingest bead fact": ClassBeadFactIngestFailed, - "fleet report: cleared comment failed": ClassSelfReportClearedFailed, - "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, - "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, - "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, - "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, - "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, - "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, - "inference no route": ClassInferenceNoRoute, - "inference upstream error": ClassInferenceUpstreamError, - "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, - "inference upstream failed": ClassInferenceUpstreamError, - "inference translate request failed": ClassInferenceUpstreamError, + "tmux has-session failed": ClassTmuxSessionCheckFailed, + "failed to send kick": ClassAgentLifecycleRepeated, + "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, + "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, + "agent tmux session missing": ClassAgentLifecycleRepeated, + "watchdog: would restart dead agent (observe mode; no action taken)": ClassAgentLifecycleRepeated, + "pane content at timeout": ClassAgentPaneTimeout, + "prompt timeout — dumping pane": ClassAgentPaneTimeout, + "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, + "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, + "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, + "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, + "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, + "copilot hung with no CLI prompt, running diagnostic": ClassCopilotCLIHung, + "diagnostic: timed out waiting for copilot error output, relaunching agent": ClassCopilotCLIHung, + "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, + "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, + "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, + "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, + "no models found from any endpoint": ClassModelsEndpointEmpty, + "login required detected": ClassAgentLoginRequired, + "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, + "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, + "watchdog: agent credential failure": ClassAgentLoginRequired, + "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, + "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, + "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, + "failed to restore interactive home mode": ClassAgentDirSetupFailed, + "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, + "failed to tighten interactive home mode": ClassAgentDirSetupFailed, + "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, + "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, + "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, + "failed to restore inference home mode": ClassAgentDirSetupFailed, + "failed to tighten inference home mode": ClassAgentDirSetupFailed, + "failed to ingest bead fact": ClassBeadFactIngestFailed, + "fleet report: cleared comment failed": ClassSelfReportClearedFailed, + "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, + "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission": ClassGitHubAppStatusesPermission, + "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, + "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, + "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, + "inference no route": ClassInferenceNoRoute, + "inference upstream error": ClassInferenceUpstreamError, + "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, + "inference upstream failed": ClassInferenceUpstreamError, + "inference translate request failed": ClassInferenceUpstreamError, } for msg, want := range fed { if got := LogFedClass(msg); got != want { @@ -1261,6 +1263,11 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { t.Errorf("%q not matched by an explicit class", msg) } } + // The neutral prow/lgtm line (5xx, 422, transport) must not file as the + // operator-only permission class; only the 403 wording names it. + if got := LogFedClass("could not set the prow/lgtm commit status"); got == ClassGitHubAppStatusesPermission { + t.Errorf("neutral prow/lgtm status failure classified as %q", got) + } for _, m := range []string{ "github REST 404 cached; suppressing repeated fetches", "copilot token validation rate-limited by GitHub; relaunching after cooldown", diff --git a/src/pkg/github/automerge_queue_prow_test.go b/src/pkg/github/automerge_queue_prow_test.go index 701ea76755..5896aef6aa 100644 --- a/src/pkg/github/automerge_queue_prow_test.go +++ b/src/pkg/github/automerge_queue_prow_test.go @@ -1,9 +1,11 @@ package github import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -186,3 +188,39 @@ func TestBindProwLgtmWarningOutlivesCallerDeadline(t *testing.T) { t.Fatalf("warning comments = %d, want 1 despite the caller's deadline expiring", rec.comments) } } + +// The permission wording is the github_app_statuses_permission_missing +// self-report keyword, so it must name only the 403; any other failure logs +// the neutral line and stays out of the operator-only permission class. +func TestBindProwLgtmWarningNamesPermissionOnlyOn403(t *testing.T) { + for _, tc := range []struct { + status int + wantPerm bool + wantPrefix string + }{ + {status: http.StatusForbidden, wantPerm: true}, + {status: http.StatusBadGateway, wantPerm: false}, + {status: http.StatusUnprocessableEntity, wantPerm: false}, + } { + t.Run(http.StatusText(tc.status), func(t *testing.T) { + rec := &prowQueueRecorder{} + api := prowQueueServer(t, "lgtm", tc.status, rec) + defer api.Close() + var logBuf bytes.Buffer + c := NewClient("token", "acme", []string{"widget"}, slog.New(slog.NewTextHandler(&logBuf, nil)), api.URL) + if err := c.bindProwLgtm(context.Background(), "acme", "widget", 7, "0123456789abcdef", "alice"); err == nil { + t.Fatal("want the status error returned") + } + logs := logBuf.String() + if !strings.Contains(logs, "could not set the prow/lgtm commit status") { + t.Fatalf("status failure not logged: %q", logs) + } + if got := strings.Contains(logs, prowLgtmStatusForbiddenMsg); got != tc.wantPerm { + t.Fatalf("permission wording present = %v, want %v: %q", got, tc.wantPerm, logs) + } + if rec.comments != 1 { + t.Fatalf("warning comments = %d, want 1", rec.comments) + } + }) + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 5e655aadd1..da0fa04b23 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -3367,8 +3367,22 @@ const ( prowLgtmWarningMarker = "<!-- hive-prow-lgtm-status-warning -->" prowWarningTimeout = 15 * time.Second + + // prowLgtmStatusForbiddenMsg is the WARN logged when GitHub answers the + // prow/lgtm status write with 403. Its text is the keyword behind the + // github_app_statuses_permission_missing self-report class + // (pkg/fleetreport/runtime.go), so it is emitted only for that case. + prowLgtmStatusForbiddenMsg = "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission" ) +// isStatusesPermissionDenied reports whether err is GitHub refusing the +// status write with 403 ("Resource not accessible by integration"), the +// signature of an App installation without Commit statuses read & write. +func isStatusesPermissionDenied(err error) bool { + var ghErr *gh.ErrorResponse + return errors.As(err, &ghErr) && ghErr.Response != nil && ghErr.Response.StatusCode == http.StatusForbidden +} + // bindProwLgtm records the `prow/lgtm` success status on headSHA, the commit // the queue review approved. On failure it returns the error, logs it when a // logger is available, and attempts to surface a warning on the PR. The caller @@ -3399,8 +3413,17 @@ func (c *Client) bindProwLgtm(ctx context.Context, owner, repoName string, numbe return nil } if c.logger != nil { - c.logger.Warn("could not set the prow/lgtm commit status; the GitHub App likely lacks the statuses: write permission", - "repo", owner+"/"+repoName, "number", number, "error", err) + // Only a 403 names the permission gap: that line is the + // github_app_statuses_permission_missing self-report keyword. Any + // other failure (5xx, 422, transport, deadline) stays neutral so a + // transient outage is not filed as an installation-permission fault. + if isStatusesPermissionDenied(err) { + c.logger.Warn(prowLgtmStatusForbiddenMsg, + "repo", owner+"/"+repoName, "number", number, "error", err) + } else { + c.logger.Warn("could not set the prow/lgtm commit status", + "repo", owner+"/"+repoName, "number", number, "error", err) + } } note := "Hive queued this PR but could not set the `prow/lgtm` commit status, so Prow may strip the `lgtm` label and the auto-merge queue may not hold." // A caller that withdrew has nothing left to warn about; a caller whose From daf71944d22e6b6776b6f559016f323801288e76 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:01:04 +0100 Subject: [PATCH 112/223] fix: avoid fleet reports for canceled stage digest reads (#17946) Hive-Run: hivecommons/hive#17944 Hive-Plan: stage digest cancellation Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> --- .../fixed-canceled-stage-digest-report.md | 1 + src/pkg/fleetreport/slogfeed/slogfeed_test.go | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 changelog.d/fixed-canceled-stage-digest-report.md diff --git a/changelog.d/fixed-canceled-stage-digest-report.md b/changelog.d/fixed-canceled-stage-digest-report.md new file mode 100644 index 0000000000..b2c11c298e --- /dev/null +++ b/changelog.d/fixed-canceled-stage-digest-report.md @@ -0,0 +1 @@ +- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 838050edbd..1474ff0dd9 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -65,6 +65,27 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestCanceledStageDigestStaysLocal(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels: check digest failed" + for range 6 { + logger.Warn(msg, "repo", "utah", "pr", 1194, "error", context.Canceled) + } + logger.Warn(msg, "error", errors.New("GET checks: context canceled")) + if len(*seen) != 0 { + t.Fatalf("canceled reads produced fleet observations: %+v", *seen) + } + if n := strings.Count(out.String(), "\n"); n != 7 { + t.Fatalf("local log records = %d, want 7", n) + } + logger.Warn(msg, "error", errors.New("context deadline exceeded")) + logger.Warn(msg, "error", errors.New("permission denied")) + logger.Warn("another failure", "error", context.Canceled) + if len(*seen) != 2 { + t.Fatalf("unexpected errors must remain reportable: %+v", *seen) + } +} + func TestCanceledAdoptionReadStaysInLog(t *testing.T) { logger, seen, out := newFeed(t) const msg = "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" From 803c1b047e692b21f81b9116aaa89139662c7ae0 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 21:30:20 +0000 Subject: [PATCH 113/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.237.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.237.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-canceled-stage-digest-report.md | 1 - changelog.d/fixed-selfreport-statuses-permission-class.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-canceled-stage-digest-report.md delete mode 100644 changelog.d/fixed-selfreport-statuses-permission-class.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b12cfc3707..fc028bac11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.237.1) + +### Fixed + +- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). +- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). + ## 2026-11-12 (v5.237.0) ### Added diff --git a/changelog.d/fixed-canceled-stage-digest-report.md b/changelog.d/fixed-canceled-stage-digest-report.md deleted file mode 100644 index b2c11c298e..0000000000 --- a/changelog.d/fixed-canceled-stage-digest-report.md +++ /dev/null @@ -1 +0,0 @@ -- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). diff --git a/changelog.d/fixed-selfreport-statuses-permission-class.md b/changelog.d/fixed-selfreport-statuses-permission-class.md deleted file mode 100644 index 84c964db2a..0000000000 --- a/changelog.d/fixed-selfreport-statuses-permission-class.md +++ /dev/null @@ -1 +0,0 @@ -- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). From 89a56e9520cf6feabf1a8e4da97bc8fce060708e Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:30:45 +0000 Subject: [PATCH 114/223] =?UTF-8?q?=F0=9F=90=9B=20fix(agent):=20don't=20WA?= =?UTF-8?q?RN-dump=20the=20pane=20when=20a=20prompt=20wait=20outlasts=20a?= =?UTF-8?q?=20live=20turn=20(#17996)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-17993-mid-turn-prompt-timeout.md | 1 + src/crux/agent/agent_working_test.go | 47 +++++++++++++++++++ src/crux/agent/manager.go | 16 +++++++ 3 files changed, 64 insertions(+) create mode 100644 changelog.d/fixed-17993-mid-turn-prompt-timeout.md diff --git a/changelog.d/fixed-17993-mid-turn-prompt-timeout.md b/changelog.d/fixed-17993-mid-turn-prompt-timeout.md new file mode 100644 index 0000000000..7e2b338b5b --- /dev/null +++ b/changelog.d/fixed-17993-mid-turn-prompt-timeout.md @@ -0,0 +1 @@ +- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). diff --git a/src/crux/agent/agent_working_test.go b/src/crux/agent/agent_working_test.go index 902143114e..8e9df60c8c 100644 --- a/src/crux/agent/agent_working_test.go +++ b/src/crux/agent/agent_working_test.go @@ -1,6 +1,9 @@ package agent import ( + "bytes" + "log/slog" + "strings" "testing" "time" @@ -67,6 +70,50 @@ func TestWaitForInputPrompt_WorkingGate(t *testing.T) { } } +// A prompt wait that times out because the agent is still mid-turn must not +// log the WARN pane dump: the slog feed files that line as agent_pane_timeout, +// so every re-kick that outwaited a long turn self-reported (#17993). A pane +// that never reaches a prompt for any other reason still dumps at WARN. +func TestWaitForInputPrompt_TimeoutWhileWorkingIsNotWarned(t *testing.T) { + origTimeout, origPoll := inputPromptTimeout, inputPromptPollInterval + inputPromptTimeout = 200 * time.Millisecond + inputPromptPollInterval = 10 * time.Millisecond + defer func() { + inputPromptTimeout, inputPromptPollInterval = origTimeout, origPoll + }() + + run := func(pane string) string { + t.Helper() + var logBuf bytes.Buffer + m := NewManager( + map[string]config.AgentConfig{"quality": {Backend: "claude"}}, + slog.New(slog.NewTextHandler(&logBuf, nil)), ProjectContext{}) + m.mu.Lock() + agent := m.agents["quality"] + m.mu.Unlock() + ft := termSeams(m) + ft.captureVisiblePane = func(*AgentProcess) string { return pane } + ft.capturePane = func(*AgentProcess) string { return pane } + if m.waitForInputPromptForAgent(agent) { + t.Fatalf("pane %q: returned ready; want not ready", pane) + } + return logBuf.String() + } + + busy := run(" ◉ Working · 42.6 KiB esc interrupt Claude Fable 5\n❯") + if strings.Contains(busy, "prompt timeout — dumping pane") { + t.Errorf("mid-turn timeout logged a WARN pane dump; want an INFO line only:\n%s", busy) + } + if !strings.Contains(busy, "prompt wait ended: agent still mid-turn") { + t.Errorf("mid-turn timeout did not log the mid-turn INFO line:\n%s", busy) + } + + stuck := run("some unrecognised screen with no prompt") + if !strings.Contains(stuck, "prompt timeout — dumping pane") { + t.Errorf("non-working timeout must still dump the pane at WARN:\n%s", stuck) + } +} + // Bob-Shell 2.x keeps the "❯" input box painted while a response streams and // signals busy only with "⠇ Processing… (Enter to steer, Tab to queue)". The // busy pane must read as working (delivery gate) and NOT as a finished turn diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 9713d21661..0d25429396 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -1242,6 +1242,8 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu defer ticker.Stop() menuEscapes := 0 dialogAnswers := 0 + // lastWorking records whether the most recent poll saw the agent mid-turn. + lastWorking := false // Inference-routed agents run Claude Code too, so a permission dialog in // their pane is answered on the same terms as a plain claude agent's. dialogBackend := backend @@ -1255,6 +1257,18 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu } select { case <-deadline: + // An agent still mid-turn at the deadline is busy, not stuck: the + // kick paths already report it as busy and leave a genuinely stalled + // turn to the busy-no-activity watchdog (#12268). Dumping the pane at + // WARN here fed the agent_pane_timeout self-report every time a + // re-kick outwaited a long turn (#17993). + if lastWorking { + m.logger.Info("prompt wait ended: agent still mid-turn", + "agent", agent.Name, + "session", agent.tmuxSession, + "waited", inputPromptTimeout) + return false, kickPromptNotReadyReason + } m.logger.Warn("prompt timeout — dumping pane", "agent", agent.Name, "session", agent.tmuxSession) @@ -1281,6 +1295,7 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu // Check the visible pane only: a dismissed consent screen // lingers in the scrollback that captureTmuxPaneForAgent sees. visible := m.captureVisiblePaneForAgent(agent) + lastWorking = false if paneShowsCopilotQuestionForm(visible) { m.dismissCopilotQuestionFormForKick(agent) continue @@ -1313,6 +1328,7 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu // Use the visible pane only for the same reason as above: a // finished task's working marker lingers in scrollback. if paneShowsAgentWorking(visible) { + lastWorking = true continue } if paneShowsInputPrompt(visible) { From a1292e749ad65d332fb045be8e910e98bce6240b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:46:22 +0000 Subject: [PATCH 115/223] =?UTF-8?q?=F0=9F=90=9B=20fix(review):=20add=20rev?= =?UTF-8?q?iew.perspectives=5Fsource=20config=20and=20ReviewPerspectives?= =?UTF-8?q?=20parser=20(#17994)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...17986-review-perspectives-source-config.md | 1 + src/pkg/config/config.go | 38 ++++++ .../config/review_perspectives_source_test.go | 109 ++++++++++++++++ .../dashboard/settings_parity_allowlist.txt | 6 + src/pkg/perspsrc/perspsrc.go | 90 ++++++++++++++ src/pkg/perspsrc/perspsrc_test.go | 116 ++++++++++++++++++ 6 files changed, 360 insertions(+) create mode 100644 changelog.d/added-17986-review-perspectives-source-config.md create mode 100644 src/pkg/config/review_perspectives_source_test.go create mode 100644 src/pkg/perspsrc/perspsrc.go create mode 100644 src/pkg/perspsrc/perspsrc_test.go diff --git a/changelog.d/added-17986-review-perspectives-source-config.md b/changelog.d/added-17986-review-perspectives-source-config.md new file mode 100644 index 0000000000..5965ebd30b --- /dev/null +++ b/changelog.d/added-17986-review-perspectives-source-config.md @@ -0,0 +1 @@ +- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 6c80ab373b..ecf4cb63f0 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -8332,6 +8332,20 @@ type ReviewConfig struct { // rather than left to infer them from the tree; a hive can only get that // by writing it down. An entry that is blank means "use the built-in". PerspectivePrompts map[string]string `yaml:"perspective_prompts,omitempty" json:"perspective_prompts,omitempty"` + // PerspectivesSource live-links the perspective names and focus text to a + // kind: ReviewPerspectives file in a GitHub repo (hivecommons/hive#17975), + // so maintainers can change the set through ordinary PRs. When it is set + // and resolved it wins over Perspectives/PerspectivePrompts, which remain + // the fallback. A partially filled source (no owner, repo or path) is + // treated as unset, matching DefinitionSourceConfig.IsSet. Only names and + // focus are sourced: combined_perspectives, plan_match and + // max_perspectives_per_pr stay operator-owned. + PerspectivesSource *DefinitionSourceConfig `yaml:"perspectives_source,omitempty" json:"perspectives_source,omitempty"` + // LinkedPerspectives is the set most recently resolved from + // PerspectivesSource. It is runtime-only and never persisted, so a fetched + // document can never be written back into the overlay. Read the effective + // set through EffectivePerspectives, never this field directly. + LinkedPerspectives *LinkedPerspectives `yaml:"-" json:"-"` // PerspectiveModels pins the backend/model one perspective's swarm review // runs with, keyed by perspective name. An entry replaces the reviewer // agent's review_models pool for that perspective only; the agent's @@ -8579,6 +8593,30 @@ type ReviewConfig struct { Reviewers ReviewersConfig `yaml:"reviewers,omitempty" json:"reviewers,omitzero"` } +// LinkedPerspectives is a review perspective set resolved from +// ReviewConfig.PerspectivesSource. It is already validated by the resolver; +// pkg/config deliberately does not import pkg/review to re-check it. +type LinkedPerspectives struct { + Names []string + Focus map[string]string + // Source is the provenance shown to operators, e.g. owner/repo/path@ref. + Source string + // Warning carries the last resolve failure while an older, still-valid set + // stays in effect. Empty when the latest document was applied. + Warning string +} + +// EffectivePerspectives returns the perspective names and focus text every +// consumer should use: the linked set when PerspectivesSource is set and has +// resolved, otherwise the inline Perspectives and PerspectivePrompts. With no +// source configured the result is exactly the inline values, unchanged. +func (r ReviewConfig) EffectivePerspectives() (names []string, focus map[string]string) { + if r.PerspectivesSource.IsSet() && r.LinkedPerspectives != nil { + return r.LinkedPerspectives.Names, r.LinkedPerspectives.Focus + } + return r.Perspectives, r.PerspectivePrompts +} + // PlanMatchConfig is the switch for the plan_match review perspective // (hivecommons/hive#8317). Off by default: the perspective only earns its cost // on a hive whose implementation PRs carry run trailers and whose plans live diff --git a/src/pkg/config/review_perspectives_source_test.go b/src/pkg/config/review_perspectives_source_test.go new file mode 100644 index 0000000000..85a7c66211 --- /dev/null +++ b/src/pkg/config/review_perspectives_source_test.go @@ -0,0 +1,109 @@ +package config + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestEffectivePerspectivesPrecedence(t *testing.T) { + source := &DefinitionSourceConfig{Owner: "acme", Repo: "widgets", Path: ".hive/review-perspectives.yaml"} + linked := &LinkedPerspectives{ + Names: []string{"correctness", "spyre-numerics"}, + Focus: map[string]string{"spyre-numerics": "fp16 accumulation order"}, + Source: "acme/widgets/.hive/review-perspectives.yaml", + } + inlineNames := []string{"security"} + inlineFocus := map[string]string{"security": "secrets in logs"} + + tests := []struct { + name string + cfg ReviewConfig + wantNames []string + wantFocus map[string]string + }{ + { + name: "source set and resolved uses linked set", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: source, LinkedPerspectives: linked}, + wantNames: linked.Names, + wantFocus: linked.Focus, + }, + { + name: "source set but unresolved falls back to inline", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: source}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "partial source is unset", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: &DefinitionSourceConfig{Owner: "acme", Repo: "widgets"}, LinkedPerspectives: linked}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "source unset uses inline", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "nothing configured", + cfg: ReviewConfig{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + names, focus := tc.cfg.EffectivePerspectives() + if !reflect.DeepEqual(names, tc.wantNames) { + t.Errorf("names = %v, want %v", names, tc.wantNames) + } + if !reflect.DeepEqual(focus, tc.wantFocus) { + t.Errorf("focus = %v, want %v", focus, tc.wantFocus) + } + }) + } +} + +func TestLinkedPerspectivesNotPersisted(t *testing.T) { + cfg := ReviewConfig{ + PerspectivesSource: &DefinitionSourceConfig{Owner: "acme", Repo: "widgets", Path: ".hive/review-perspectives.yaml"}, + LinkedPerspectives: &LinkedPerspectives{ + Names: []string{"linkedonlyname"}, + Focus: map[string]string{"linkedonlyname": "linkedonlyfocus"}, + Source: "linkedonlyprovenance", + Warning: "linkedonlywarning", + }, + } + y, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + j, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + for format, out := range map[string]string{"yaml": string(y), "json": string(j)} { + if !strings.Contains(out, "perspectives_source") { + t.Errorf("%s output lost perspectives_source:\n%s", format, out) + } + for _, leaked := range []string{"linked", "Linked"} { + if strings.Contains(out, leaked) { + t.Errorf("%s output persists linked data (%q):\n%s", format, leaked, out) + } + } + } + + var round ReviewConfig + if err := yaml.Unmarshal(y, &round); err != nil { + t.Fatalf("yaml.Unmarshal: %v", err) + } + if round.LinkedPerspectives != nil { + t.Errorf("LinkedPerspectives survived a yaml round-trip: %+v", round.LinkedPerspectives) + } + if !round.PerspectivesSource.IsSet() { + t.Errorf("PerspectivesSource lost in yaml round-trip: %+v", round.PerspectivesSource) + } +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 0c0c2c8f71..3bf0e75a3c 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -516,6 +516,12 @@ review.perspective_models.<key>.backend # TODO #13243 review.perspective_models.<key>.model # TODO #13243 review.perspective_prompts.<key> # TODO #11854 review.perspectives[] # TODO #11854 +review.perspectives_source.owner # TODO #17975 +review.perspectives_source.path # TODO #17975 +review.perspectives_source.ref # TODO #17975 +review.perspectives_source.repo # TODO #17975 +review.perspectives_source.type # TODO #17975 +review.perspectives_source.url # TODO #17975 review.recommendations.labels[] # TODO #11854 review.recommendations.repos[] # TODO #11854 review.reviewer_agents[] # TODO #11854 diff --git a/src/pkg/perspsrc/perspsrc.go b/src/pkg/perspsrc/perspsrc.go new file mode 100644 index 0000000000..0d03c4bc68 --- /dev/null +++ b/src/pkg/perspsrc/perspsrc.go @@ -0,0 +1,90 @@ +// Package perspsrc parses a live-linked review perspective set: a +// kind: ReviewPerspectives YAML document kept in a GitHub repo and named by +// review.perspectives_source (hivecommons/hive#17975), so maintainers can add, +// remove and re-describe perspectives through ordinary PRs. +// +// It is the review-set analogue of pkg/defsrc (which live-sources a whole +// agent definition) and follows the same design principles: +// +// - Validate before use: a document is only accepted when it would load as +// inline config would, via review.NewPerspectiveSet. A document that would +// not is an error, never a silent fall back to the built-in set. +// - Bounded input: a document over maxDocumentBytes is rejected outright. +// +// The document covers perspective names and focus text only. Cost and +// dispatch settings (combined_perspectives, plan_match, +// max_perspectives_per_pr) stay operator-owned in the hive config. +// +// This package, not pkg/config, owns the dependency on pkg/review, so +// pkg/config never has to import pkg/review. +package perspsrc + +import ( + "fmt" + "strings" + + "github.com/hivecommons/hive/pkg/review" + "gopkg.in/yaml.v3" +) + +// maxDocumentBytes caps how large a ReviewPerspectives document may be. A real +// one is a short list (well under a kilobyte per perspective); this bounds +// memory/log blast radius if a source path points at something huge. +const maxDocumentBytes = 512 * 1024 // 512 KiB + +// Kind is the required `kind` value of a review perspective document. +const Kind = "ReviewPerspectives" + +// Document is the YAML format of a live-linked review perspective set. +type Document struct { + Kind string `yaml:"kind" json:"kind"` + Perspectives []Entry `yaml:"perspectives" json:"perspectives"` +} + +// Entry is one perspective. Focus is optional for a built-in perspective +// (empty keeps the built-in text) and required for a hive-defined one. +type Entry struct { + Name string `yaml:"name" json:"name"` + Focus string `yaml:"focus,omitempty" json:"focus,omitempty"` +} + +// ParseDocument parses and validates a ReviewPerspectives document, returning +// the perspective names in dispatch order and the focus text keyed by name, in +// the same shape as review.perspectives and review.perspective_prompts. +func ParseDocument(doc string) (names []string, focus map[string]string, err error) { + if len(doc) > maxDocumentBytes { + return nil, nil, fmt.Errorf("perspectives document is %d bytes, over the limit of %d", len(doc), maxDocumentBytes) + } + var d Document + if err := yaml.Unmarshal([]byte(doc), &d); err != nil { + return nil, nil, fmt.Errorf("invalid perspectives YAML: %w", err) + } + if d.Kind != Kind { + return nil, nil, fmt.Errorf("expected kind %q, got %q", Kind, d.Kind) + } + if len(d.Perspectives) == 0 { + return nil, nil, fmt.Errorf("perspectives list is empty") + } + names = make([]string, 0, len(d.Perspectives)) + focus = map[string]string{} + seen := map[string]bool{} + for i, e := range d.Perspectives { + name := strings.TrimSpace(e.Name) + if name == "" { + return nil, nil, fmt.Errorf("perspectives[%d].name is required", i) + } + key := strings.ToLower(name) + if seen[key] { + return nil, nil, fmt.Errorf("duplicate perspective %q", name) + } + seen[key] = true + names = append(names, name) + if f := strings.TrimSpace(e.Focus); f != "" { + focus[name] = f + } + } + if _, err := review.NewPerspectiveSet(names, focus); err != nil { + return nil, nil, err + } + return names, focus, nil +} diff --git a/src/pkg/perspsrc/perspsrc_test.go b/src/pkg/perspsrc/perspsrc_test.go new file mode 100644 index 0000000000..ae352b834a --- /dev/null +++ b/src/pkg/perspsrc/perspsrc_test.go @@ -0,0 +1,116 @@ +package perspsrc + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseDocument(t *testing.T) { + tests := []struct { + name string + doc string + wantNames []string + wantFocus map[string]string + wantErr string + }{ + { + name: "built-in only", + doc: `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: security +`, + wantNames: []string{"correctness", "security"}, + wantFocus: map[string]string{}, + }, + { + name: "custom with focus", + doc: `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: spyre-constraints + focus: "Head size a multiple of 64; fp16 only." +`, + wantNames: []string{"correctness", "spyre-constraints"}, + wantFocus: map[string]string{"spyre-constraints": "Head size a multiple of 64; fp16 only."}, + }, + { + name: "built-in with focus override", + doc: `kind: ReviewPerspectives +perspectives: + - name: style + focus: "Follow the repo's CONTRIBUTING.md conventions." +`, + wantNames: []string{"style"}, + wantFocus: map[string]string{"style": "Follow the repo's CONTRIBUTING.md conventions."}, + }, + { + name: "wrong kind", + doc: `kind: AgentDefinition +perspectives: + - name: correctness +`, + wantErr: `expected kind "ReviewPerspectives"`, + }, + { + name: "missing kind", + doc: `perspectives: + - name: correctness +`, + wantErr: `expected kind "ReviewPerspectives", got ""`, + }, + { + name: "empty list", + doc: "kind: ReviewPerspectives\nperspectives: []\n", + wantErr: "perspectives list is empty", + }, + { + name: "unknown name without focus", + doc: `kind: ReviewPerspectives +perspectives: + - name: sekurity +`, + wantErr: "has no focus text", + }, + { + name: "duplicate name", + doc: `kind: ReviewPerspectives +perspectives: + - name: security + - name: Security +`, + wantErr: "duplicate perspective", + }, + { + name: "oversize", + doc: "kind: ReviewPerspectives\n# " + strings.Repeat("x", maxDocumentBytes) + "\nperspectives:\n - name: correctness\n", + wantErr: "over the limit", + }, + { + name: "invalid yaml", + doc: "kind: [unterminated", + wantErr: "invalid perspectives YAML", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + names, focus, err := ParseDocument(tc.doc) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ParseDocument() error = %v, want containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("ParseDocument() unexpected error: %v", err) + } + if !reflect.DeepEqual(names, tc.wantNames) { + t.Errorf("names = %v, want %v", names, tc.wantNames) + } + if !reflect.DeepEqual(focus, tc.wantFocus) { + t.Errorf("focus = %v, want %v", focus, tc.wantFocus) + } + }) + } +} From c8ff67f7b87899451e6a98bacb1300ae08c924a8 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:04:48 +0100 Subject: [PATCH 116/223] =?UTF-8?q?=F0=9F=90=9B=20Avoid=20self-reporting?= =?UTF-8?q?=20intentional=20signing=20skips=20(#17928)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Avoid self-reporting intentional signing skips Hive-Run: hivecommons/hive#17924 Hive-Plan: signed-commit skip self-report Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * docs: explain signed-commit ownership suppression Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-signed-commit-skip-selfreport.md | 1 + src/docs/fleet-report.md | 10 ++++++++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 9 ++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 19 +++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-signed-commit-skip-selfreport.md diff --git a/changelog.d/fixed-signed-commit-skip-selfreport.md b/changelog.d/fixed-signed-commit-skip-selfreport.md new file mode 100644 index 0000000000..cbf6f24ade --- /dev/null +++ b/changelog.d/fixed-signed-commit-skip-selfreport.md @@ -0,0 +1 @@ +- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index bd63e9662b..db5eb42f78 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -246,6 +246,16 @@ nudge-delivery failures remain eligible for reporting when repeated. It also skips `stage transition suppressed by dampener` (`stageDampenerSuppression`, `pkg/fleetreport/slogfeed/slogfeed.go`): the stage-label store suppresses flapping transitions during its dampener hold window (hivecommons/hive#17103). That is the dampener working as designed, not a runtime error. The same function skips `stage flap suppressed` (`suppressStageFlap`, `pkg/stageengine/writer.go`): the stage engine refuses a same-head, same-evidence reversal of its last label write inside its flap window (hivecommons/hive#17142). +It also skips the exact WARN `signed-commit reconciler: skipped; the PR keeps an +unsigned commit` (`signedCommitOwnershipSkip`, `pkg/fleetreport/slogfeed/slogfeed.go`; +emitted by `pkg/github/pr_signed_reconcile.go`): the signed-commit reconciler +deliberately refuses to rewrite a merge commit or a commit authored or committed +by anyone other than a hive agent, and records the remedy on the PR +(`noteSignedSkip`). Repeated refusals are the ownership guard working as +designed, not a repeated runtime defect (hivecommons/hive#17924). The WARN and +its `reason` stay in local logs; other reconciler failures and the same message +at ERROR remain reportable. + The re-engagement sweep logs `re-engagement kick undeliverable; budget not charged` at INFO rather than WARN when the cause is `shared session pool is busy` (`deliverReEngagement`, `src/cmd/hive/main_sweeps.go`): a saturated shared CLI diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index e35e8781fd..5a9507e578 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -115,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -136,6 +136,13 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } +// signedCommitOwnershipSkip is the reconciler's deliberate refusal to rewrite +// a foreign or merge commit. The warning and PR remedy remain visible; repeated +// skips across PRs are not a runtime defect. +func signedCommitOwnershipSkip(r slog.Record) bool { + return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == "signed-commit reconciler: skipped; the PR keeps an unsigned commit" +} + // canceledError applies the existing cancellation exclusion to structured // errors too. An accompanying different error must remain reportable. func canceledError(r slog.Record, bound []slog.Attr) bool { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 1474ff0dd9..d21d195f56 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -65,6 +65,25 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestSignedCommitOwnershipSkipStaysInLogsWithoutFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "signed-commit reconciler: skipped; the PR keeps an unsigned commit" + for range 6 { + logger.Warn(msg, "repo", "o/r", "pr", 1403, "reason", "commit is authored by another account") + } + if len(*seen) != 0 { + t.Fatalf("expected ownership skips produced observations: %+v", *seen) + } + if n := strings.Count(out.String(), msg); n != 6 || !strings.Contains(out.String(), "another account") { + t.Fatalf("local logs lost the warning or its reason: %s", out.String()) + } + logger.Warn("signed-commit reconciler: listing PR commits failed", "err", "unexpected failure") + logger.Error(msg, "reason", "unexpected failure") + if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + func TestCanceledStageDigestStaysLocal(t *testing.T) { logger, seen, out := newFeed(t) const msg = "stage labels: check digest failed" From acb831821b79a852ef0b243fb4191ed4b084f4d4 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 22:07:53 +0000 Subject: [PATCH 117/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.237.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.237.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-17993-mid-turn-prompt-timeout.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-17993-mid-turn-prompt-timeout.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fc028bac11..f12805c14c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.237.2) + +### Fixed + +- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). + ## 2026-11-12 (v5.237.1) ### Fixed diff --git a/changelog.d/fixed-17993-mid-turn-prompt-timeout.md b/changelog.d/fixed-17993-mid-turn-prompt-timeout.md deleted file mode 100644 index 7e2b338b5b..0000000000 --- a/changelog.d/fixed-17993-mid-turn-prompt-timeout.md +++ /dev/null @@ -1 +0,0 @@ -- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). From 4d874403518e7261cfd40eac7b3ade7bdcc9a572 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:20:13 +0000 Subject: [PATCH 118/223] fix(dashboard): summarize dropped SSE events instead of warning per event (#17998) * fix(dashboard): summarize dropped SSE events instead of warning per event Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> * docs(api-reference): refresh server.go citations after SSE drop summary Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-sse-drop-warn-flood.md | 1 + src/docs/api-reference/auth.md | 6 ++--- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++++++++---------- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/dashboard/server.go | 21 +++++++++++++++++- src/pkg/dashboard/server_test.go | 21 ++++++++++++++++++ 7 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 changelog.d/fixed-sse-drop-warn-flood.md diff --git a/changelog.d/fixed-sse-drop-warn-flood.md b/changelog.d/fixed-sse-drop-warn-flood.md new file mode 100644 index 0000000000..a5494ff99b --- /dev/null +++ b/changelog.d/fixed-sse-drop-warn-flood.md @@ -0,0 +1 @@ +- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 1d3e6c8eff..83820127f5 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1688` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1689` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1690` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1693` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1694` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1695` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 8511ded0f0..96f6d9c44f 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1695` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1700` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 85fa9acf7b..acefbb9f3c 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1668` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1669` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1670` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1676` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1682` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1683` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1684` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1685` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1686` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1687` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1673` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1674` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1675` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1686` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1687` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1688` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1689` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1690` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1691` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1692` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index c3da74dff5..d6909a3512 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1758` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1763` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 77b80d02e8..fd00f40ddb 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -64,6 +64,9 @@ const agentSkipAfterFullBroadcastS = 5 * time.Second const maxSSEClients = 100 +// sseDropWarnInterval bounds how often dropped SSE events are logged at WARN. +const sseDropWarnInterval = 5 * time.Minute + const sessionCookieName = "hive_session" const sessionCookieMaxAge = 30 * 24 * 60 * 60 @@ -139,6 +142,8 @@ type Server struct { status *StatusPayload sseClients map[chan []byte]struct{} sseMu sync.Mutex + sseDropped int // events dropped since the last summary warning; guarded by sseMu + sseDropWarn time.Time // last summary warning; guarded by sseMu logger *slog.Logger mux *http.ServeMux deps *Dependencies @@ -4049,9 +4054,23 @@ func (s *Server) broadcastFrame(frame string) { select { case ch <- raw: default: - s.logger.Warn("SSE client too slow, dropping event") + s.sseDropped++ + s.logger.Debug("SSE client too slow, dropping event") } } + s.warnSSEDropsLocked(time.Now()) +} + +// warnSSEDropsLocked emits one summary warning per sseDropWarnInterval instead +// of one per dropped event, so a single stalled browser tab does not flood logs +// or trip repeated-error fleet reporting. Caller holds sseMu. +func (s *Server) warnSSEDropsLocked(now time.Time) { + if s.sseDropped == 0 || now.Sub(s.sseDropWarn) < sseDropWarnInterval { + return + } + s.logger.Warn("SSE clients too slow, dropped events", "dropped", s.sseDropped, "interval", sseDropWarnInterval.String()) + s.sseDropped = 0 + s.sseDropWarn = now } // initHistories lazily builds the three sparkline ring buffers. Called (once) diff --git a/src/pkg/dashboard/server_test.go b/src/pkg/dashboard/server_test.go index 522f7bfa06..a0fe9a0704 100644 --- a/src/pkg/dashboard/server_test.go +++ b/src/pkg/dashboard/server_test.go @@ -2,6 +2,7 @@ package dashboard import ( "bufio" + "bytes" "encoding/json" "fmt" "log/slog" @@ -905,3 +906,23 @@ func TestNoAuth_AllEndpointsAccessible(t *testing.T) { t.Errorf("expected 200 when auth is not configured, got %d", resp.StatusCode) } } + +func TestBroadcast_SlowClientDropsWarnOncePerInterval(t *testing.T) { + var buf bytes.Buffer + s := newTestServer() + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + full := make(chan []byte, 1) + full <- []byte("old-data") + s.sseMu.Lock() + s.sseClients[full] = struct{}{} + s.sseMu.Unlock() + + for i := 0; i < 20; i++ { + s.broadcastFrame("data: {}\n\n") + } + + if got := strings.Count(buf.String(), "level=WARN"); got != 1 { + t.Errorf("WARN lines: want 1, got %d\n%s", got, buf.String()) + } +} From 6edcba9695c04c8c2a9aa773e2e8d20b08f974f4 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 15:37:24 -0700 Subject: [PATCH 119/223] =?UTF-8?q?=F0=9F=90=9B=20Claim=20onto=20a=20non-d?= =?UTF-8?q?efault=20forge:=20show=20the=20selected=20forge=20while=20pendi?= =?UTF-8?q?ng=20and=20discover=20the=20App=20installation=20on=20it=20(#18?= =?UTF-8?q?007)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Show operator-selected forge on claim-pending fleet rows While a placeholder is assigned-but-unclaimed the spoke still heartbeats the pool default github_host (github.ibm.com on vllm-d), which the hub treated as authoritative and rendered over the explicit forge the operator picked in the assign dialog. Prefer the hive record's github_host for AssignedUnclaimed rows; the spoke report resumes authority once the claim is delivered. Fixes #18006 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 Move App installation discovery to the hub-delivered App on forge switch A claim that switches forge (GHE pool default → github.com) delivers the github.com App with installation_id 0. HasUsableApp is false for that identity so the client rebuild was skipped — correctly — but installation discovery kept the boot-time AppAuth (App 5686 @ github.ibm.com) and polled the wrong forge forever. Build a discovery-only AppAuth for the new App and hand it to the dashboard so the owner's installation is adopted. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- .../fixed-claim-forge-switch-discovery.md | 1 + changelog.d/fixed-claim-pending-forge-pill.md | 1 + .../discovery_appauth_forge_switch_test.go | 54 ++++++++++++++++ src/cmd/hive/main.go | 10 +++ src/cmd/hive/main_github_app.go | 54 ++++++++++++++++ src/pkg/dashboard/server.go | 10 +++ src/pkg/hub/claim_pending_forge_pill_test.go | 62 +++++++++++++++++++ src/pkg/hub/saas_hives_api.go | 25 ++++++++ 8 files changed, 217 insertions(+) create mode 100644 changelog.d/fixed-claim-forge-switch-discovery.md create mode 100644 changelog.d/fixed-claim-pending-forge-pill.md create mode 100644 src/cmd/hive/discovery_appauth_forge_switch_test.go create mode 100644 src/pkg/hub/claim_pending_forge_pill_test.go diff --git a/changelog.d/fixed-claim-forge-switch-discovery.md b/changelog.d/fixed-claim-forge-switch-discovery.md new file mode 100644 index 0000000000..788c262af6 --- /dev/null +++ b/changelog.d/fixed-claim-forge-switch-discovery.md @@ -0,0 +1 @@ +- Spoke: a placeholder claimed onto a different forge than its pool default (GHE slot → github.com repo) now moves GitHub App installation discovery to the delivered App and API URL even while installation_id is still 0, instead of polling the boot-time App/forge forever and reporting "App 5686 has no installation" until a pod restart. diff --git a/changelog.d/fixed-claim-pending-forge-pill.md b/changelog.d/fixed-claim-pending-forge-pill.md new file mode 100644 index 0000000000..c5b31e94d6 --- /dev/null +++ b/changelog.d/fixed-claim-pending-forge-pill.md @@ -0,0 +1 @@ +- Hub fleet view: a placeholder assigned with an explicit forge now shows that forge (e.g. github.com) in the Location pill while the claim is pending, instead of the cluster default the spoke still heartbeats from its pre-claim config (github.ibm.com on vllm-d). diff --git a/src/cmd/hive/discovery_appauth_forge_switch_test.go b/src/cmd/hive/discovery_appauth_forge_switch_test.go new file mode 100644 index 0000000000..c9a946d56f --- /dev/null +++ b/src/cmd/hive/discovery_appauth_forge_switch_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// A placeholder booted against its pool's default forge (GHE App 5686) and +// then claimed onto github.com receives App 3568013 with installation_id 0. +// HasUsableApp is false for that identity, so the full client rebuild is +// (correctly) skipped — but installation discovery must still move to the new +// App/forge, or it polls the old one forever and the hive never adopts the +// installation its owner makes (live: `ash` → thinkscientist on vllm-d). +func TestDiscoveryAppAuthStaleOnForgeSwitchWithoutInstallation(t *testing.T) { + gheCfg := &config.Config{} + gheCfg.GitHub.AppID = 5686 + gheCfg.GitHub.APIURL = "https://github.ibm.com/api/v3" + booted := testAppAuth(t, gheCfg) + + public := config.GitHubConfig{AppID: 3568013, InstallationID: 0, APIURL: "https://api.github.com"} + + if !discoveryAppAuthStale(booted, public, "/data/gh-app-key-3568013.pem") { + t.Fatal("GHE-booted AppAuth must be replaced when the hub delivers the github.com App with no installation") + } + if !discoveryAppAuthStale(nil, public, "/data/gh-app-key-3568013.pem") { + t.Fatal("a spoke that booted with no AppAuth must get one for discovery") + } +} + +func TestDiscoveryAppAuthStaleNoOpCases(t *testing.T) { + cfg := &config.Config{} + cfg.GitHub.AppID = 3568013 + cfg.GitHub.APIURL = "https://api.github.com" + current := testAppAuth(t, cfg) + + cases := []struct { + name string + gh config.GitHubConfig + key string + }{ + {"same App and forge already held", config.GitHubConfig{AppID: 3568013, APIURL: "https://api.github.com/"}, "/k.pem"}, + {"installation known — the usable-App rebuild owns this", config.GitHubConfig{AppID: 5686, InstallationID: 42, APIURL: "https://github.ibm.com/api/v3"}, "/k.pem"}, + {"no key resolved", config.GitHubConfig{AppID: 5686, APIURL: "https://github.ibm.com/api/v3"}, ""}, + {"placeholder App is not a real App", config.GitHubConfig{AppID: config.PlaceholderAppID}, "/k.pem"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if discoveryAppAuthStale(current, tc.gh, tc.key) { + t.Errorf("expected no discovery AppAuth swap") + } + }) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 4e643473d5..ea1821f5e2 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5495,6 +5495,16 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { "app_id", b.cfg.GitHub.AppID, "installation_id", b.cfg.GitHub.InstallationID, ) + } else if discoveryAppAuthStale(b.appAuth, b.cfg.GitHub, rebuildKeyFile) { + // A real App with no installation yet (forge switch on claim): + // give installation discovery the NEW App's JWT so it polls + // the forge the repos actually live on. + newAppAuth, err := github.NewAppAuth(b.cfg.GitHub.AppID, 0, rebuildKeyFile, b.logger, b.cfg.GitHub.ResolvedAPIURL()) + if err != nil { + b.logger.Error("github app auth init for installation discovery failed", "error", err) + return + } + b.adoptDiscoveryAppAuth(newAppAuth) } }), diff --git a/src/cmd/hive/main_github_app.go b/src/cmd/hive/main_github_app.go index b747bdca20..a3a89f0f26 100644 --- a/src/cmd/hive/main_github_app.go +++ b/src/cmd/hive/main_github_app.go @@ -558,3 +558,57 @@ func advisoryIssueMissingError(repo string, cause error) string { } return base } + +// discoveryAppAuthStale reports whether the AppAuth the installation-discovery +// loop is polling with no longer names the App the config now claims. +// +// A hub-delivered identity with installation_id 0 (a placeholder claimed onto +// a DIFFERENT forge than its pool default: GHE slot → github.com repo) is not +// "usable" — HasUsableApp is false — so the heartbeat rebuild skipped it and +// the boot-time AppAuth (old App, old api_url) kept serving auto-discovery. +// Discovery then polled the wrong forge forever: the installation the owner +// made on the right one could never be found, and the hive sat at "App <old +// id> has no installation" until a pod restart. Boot builds an AppAuth the +// moment a real app_id and key exist precisely so discovery has a JWT to use; +// this is the same rule applied to a hub delivery. +func discoveryAppAuthStale(current *github.AppAuth, gh config.GitHubConfig, keyFile string) bool { + if !gh.HasApp() || gh.InstallationID != 0 || keyFile == "" { + return false + } + if current == nil { + return true + } + return current.AppID() != gh.AppID || !sameAPIURL(current.APIURL(), gh.ResolvedAPIURL()) +} + +// sameAPIURL compares two GitHub API base URLs ignoring scheme-case and a +// trailing slash, the two spellings the config and the client disagree on. +func sameAPIURL(a, b string) bool { + norm := func(s string) string { return strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/") } + return norm(a) == norm(b) +} + +// adoptDiscoveryAppAuth hands a not-yet-installed App's AppAuth to the +// installation-discovery loop and flips the banner/hub state to +// not-installed for the App it now names, then forces one discovery pass so +// an installation already made on the new forge is adopted without waiting +// for the next tick. No client is built: there is no installation to mint +// against yet — AutoDiscoverGitHubInstallationID builds it on success. +func (b *boot) adoptDiscoveryAppAuth(newAppAuth *github.AppAuth) { + if b == nil || newAppAuth == nil { + return + } + b.appAuth = newAppAuth + diag := "GitHub App " + strconv.FormatInt(b.cfg.GitHub.AppID, 10) + + " has no installation for this org — install it (the spoke adopts the installation automatically)" + b.appAuthFailure = "The GitHub App is configured but has no installation. Install the app on your org; this hive will discover the installation automatically." + if b.dashSrv != nil { + b.dashSrv.UpdateGitHubAppAuth(newAppAuth) + b.dashSrv.SetGitHubAppRequired(true) + b.dashSrv.SetGitHubAppState(github.AppStateNotInstalled.String()) + b.dashSrv.SetGitHubAppPermIssue(diag) + go func() { _, _ = b.dashSrv.AutoDiscoverGitHubInstallationID(b.ctx, true) }() + } + b.logger.Info("github app identity adopted for installation discovery", + "app_id", newAppAuth.AppID(), "api_url", newAppAuth.APIURL()) +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index fd00f40ddb..d53f4cc24b 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -3176,6 +3176,16 @@ func (s *Server) UpdateGitHubClient(client *github.Client, auth *github.AppAuth) } } +// UpdateGitHubAppAuth swaps only the App credential the installation-discovery +// loop polls with, leaving the client untouched. Used when a hub delivery names +// a new App (or forge) whose installation is not known yet, so discovery asks +// the right forge instead of the one the pod booted against. +func (s *Server) UpdateGitHubAppAuth(auth *github.AppAuth) { + if s.deps != nil { + s.deps.GHAppAuth = auth + } +} + // SetGoodFirstIssueStats installs the provider of good-first-issue curation // counters; nil removes it. func (s *Server) SetGoodFirstIssueStats(fn func() any) { diff --git a/src/pkg/hub/claim_pending_forge_pill_test.go b/src/pkg/hub/claim_pending_forge_pill_test.go new file mode 100644 index 0000000000..2f96d13ee1 --- /dev/null +++ b/src/pkg/hub/claim_pending_forge_pill_test.go @@ -0,0 +1,62 @@ +package hub + +import "testing" + +// A placeholder assigned with an explicit forge must show THAT forge in the +// Location pill while the claim is pending, not the cluster default the spoke +// still heartbeats from its pre-claim config (vllm-d: github.ibm.com). Live +// example: "ash" assigned to thinkscientist with github.com selected read +// github.ibm.com for the whole "claim pending" window. +func TestClaimPendingGitHubHostOverridesSpokeReport(t *testing.T) { + cases := []struct { + name string + entry MyHiveEntry + sh *SaaSHive + wantHost string + wantApply bool + }{ + { + name: "pending claim with explicit github.com beats GHE heartbeat", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.ibm.com"}, AssignedUnclaimed: true}, + sh: &SaaSHive{Status: statusAssigned, GitHubHost: "github.com"}, + wantHost: "github.com", + wantApply: true, + }, + { + name: "pending claim with explicit GHE beats public heartbeat", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.com"}, AssignedUnclaimed: true}, + sh: &SaaSHive{Status: statusAssigned, GitHubHost: "https://github.ibm.com/"}, + wantHost: "github.ibm.com", + wantApply: true, + }, + { + name: "delivered claim leaves the spoke report authoritative", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.ibm.com"}, AssignedUnclaimed: false}, + sh: &SaaSHive{Status: statusAssigned, ClaimDelivered: true, GitHubHost: "github.com"}, + wantApply: false, + }, + { + name: "pending claim without a recorded host does not override", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.ibm.com"}, AssignedUnclaimed: true}, + sh: &SaaSHive{Status: statusAssigned}, + wantApply: false, + }, + { + name: "no hive record does not override", + entry: MyHiveEntry{AssignedUnclaimed: true}, + sh: nil, + wantApply: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := claimPendingGitHubHost(&tc.entry, tc.sh) + if ok != tc.wantApply { + t.Fatalf("apply = %v, want %v", ok, tc.wantApply) + } + if ok && got != tc.wantHost { + t.Errorf("host = %q, want %q", got, tc.wantHost) + } + }) + } +} diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index a6713566dc..f71da247f5 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -340,6 +340,27 @@ type MyHiveEntry struct { Quadrant *Quadrant `json:"quadrant,omitempty"` } +// claimPendingGitHubHost returns the GitHub host the Location pill should show +// for an assigned-but-unclaimed row, and whether that override applies. +// +// While a placeholder is "claim pending" the spoke still runs the pool's +// default config, so its heartbeat reports the CLUSTER default (github.ibm.com +// on vllm-d) even though the operator just picked github.com in the assign +// dialog. For that window the hive record's github_host — the operator's +// explicit choice, stamped by handleAssignHive — is the truth the row must +// show; the spoke's value is simply the stale pre-claim config, not evidence +// of where the assigned repos live. Once the claim is delivered the spoke +// report resumes its authority (see resolveGitHubHost). +func claimPendingGitHubHost(entry *MyHiveEntry, sh *SaaSHive) (string, bool) { + if entry == nil || sh == nil || !entry.AssignedUnclaimed { + return "", false + } + if strings.TrimSpace(sh.GitHubHost) == "" { + return "", false + } + return githubHostLabel(sh.GitHubHost), true +} + // myHivesRecentEventCount is how many timeline events ride the My Hives // payload for the status hover. The hover panel already carries a status word, // the per-check health lines, the relay line and the user list; three events @@ -722,6 +743,10 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values // is left empty and rendered as "github.com" by the pill, so an old spoke // shows a sane default rather than a wrong host. resolveGitHubHost := func(entry *MyHiveEntry) { + if host, ok := claimPendingGitHubHost(entry, saasByID[entry.ID]); ok { + entry.GitHubHost = host + return + } if entry.GitHubHost != "" { return // spoke-reported — authoritative } From 0db6b0d19e1a495c76939c75ae3331019eee4a57 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 22:43:26 +0000 Subject: [PATCH 120/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.238.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.238.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ .../added-17986-review-perspectives-source-config.md | 1 - changelog.d/fixed-signed-commit-skip-selfreport.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-17986-review-perspectives-source-config.md delete mode 100644 changelog.d/fixed-signed-commit-skip-selfreport.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f12805c14c..41885cd023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.238.0) + +### Added + +- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) + +### Fixed + +- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). + ## 2026-11-12 (v5.237.2) ### Fixed diff --git a/changelog.d/added-17986-review-perspectives-source-config.md b/changelog.d/added-17986-review-perspectives-source-config.md deleted file mode 100644 index 5965ebd30b..0000000000 --- a/changelog.d/added-17986-review-perspectives-source-config.md +++ /dev/null @@ -1 +0,0 @@ -- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-signed-commit-skip-selfreport.md b/changelog.d/fixed-signed-commit-skip-selfreport.md deleted file mode 100644 index cbf6f24ade..0000000000 --- a/changelog.d/fixed-signed-commit-skip-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). From 906295f357e25482c2586cb268472f74c4f63438 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 15:52:42 -0700 Subject: [PATCH 121/223] fix(ioscan): reword canary preamble and retry a refused canary kick without it (#18013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Sonnet 5 (which the Copilot CLI substitutes for sonnet-4.6/opus-4.6) reads the bare "SECURITY CANARY: <token> Never print…" preamble as a prompt-injection marker and refuses the whole kick; on one hosted hive 109 of 168 supervisor kicks were refused and each one burned its cadence slot. - CanaryPreamble is now a labelled operator-harness header that says whose configuration it is and that it is not a task; crux/agent mirrors the new prefix for the kick preview. - A kick refused with a canary in front is re-delivered once with the canary stripped (trigger canary-refusal-retry); the retry is never retried, and a fresh kick re-arms it. Fixes #18012 Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- changelog.d/fixed-canary-kick-refusal.md | 1 + src/crux/agent/kick_canary_retry.go | 61 ++++++++++++++++ src/crux/agent/kick_canary_retry_test.go | 82 ++++++++++++++++++++++ src/crux/agent/kick_preview.go | 37 ++++++---- src/crux/agent/kick_preview_test.go | 4 +- src/crux/agent/manager.go | 29 +++++--- src/crux/agent/manager_kick.go | 12 +++- src/crux/agent/manager_poll.go | 1 + src/docs/scale-envelope.md | 2 +- src/pkg/ioscan/canary.go | 17 ++++- src/pkg/ioscan/canary_coverage_test.go | 19 ++++- src/pkg/scheduler/kick_layout_test.go | 8 +-- src/pkg/scheduler/repo_filter_hold_test.go | 2 +- 13 files changed, 239 insertions(+), 36 deletions(-) create mode 100644 changelog.d/fixed-canary-kick-refusal.md create mode 100644 src/crux/agent/kick_canary_retry.go create mode 100644 src/crux/agent/kick_canary_retry_test.go diff --git a/changelog.d/fixed-canary-kick-refusal.md b/changelog.d/fixed-canary-kick-refusal.md new file mode 100644 index 0000000000..5c1235ffe5 --- /dev/null +++ b/changelog.d/fixed-canary-kick-refusal.md @@ -0,0 +1 @@ +- Kicks are no longer refused wholesale as "prompt injection" because of the ioscan exfiltration canary: the preamble is now a labelled `=== HIVE HARNESS HEADER` block that states it is the operator's own configuration and not a task (Claude Sonnet 5 via the Copilot CLI refused ~65% of one hosted supervisor's kicks on the old `SECURITY CANARY:` marker), and a kick that is still refused with a canary in front is re-delivered once without it instead of burning the cadence slot (#18012). diff --git a/src/crux/agent/kick_canary_retry.go b/src/crux/agent/kick_canary_retry.go new file mode 100644 index 0000000000..99eb8be642 --- /dev/null +++ b/src/crux/agent/kick_canary_retry.go @@ -0,0 +1,61 @@ +package agent + +// A kick that opens with the ioscan exfiltration canary can be refused as a +// whole by a model that reads the canary as a prompt-injection marker +// (Claude Sonnet 5 via the Copilot CLI: "this request includes a 'SECURITY +// CANARY' marker and asks me to act as a scanner agent …"). The refusal used +// to cost the entire cadence slot — a 4h scanner kick, or ~150 five-minute +// supervisor kicks a day on one hosted hive. The refused kick is instead +// re-delivered once with the canary preamble stripped; the retry is a plain +// kick, so a second refusal is the model's verdict on the work itself and +// stands. + +// kickTriggerCanaryRetry is the delivery trigger of the one canary-free +// re-delivery a refused canary kick gets. recordDeliveredKickLocked uses it +// to mark the retry so it is never itself retried. +const kickTriggerCanaryRetry = "canary-refusal-retry" + +// planCanaryRetry decides from the refused kick alone whether it earns a +// canary-free re-delivery: the kick as handed to delivery must have carried +// a canary preamble with a body after it, and must not already be the retry. +func planCanaryRetry(lastKickInput string, alreadyRetried bool) (plain string, ok bool) { + if alreadyRetried { + return "", false + } + plain, had := stripCanaryPreamble(lastKickInput) + if !had { + return "", false + } + return plain, true +} + +// retryRefusedCanaryKick is the checkKickRefusal hook: when the kick just +// refused led with a canary, re-deliver it once without one. Delivery goes +// through the full SendKick readiness chain on its own goroutine (the caller +// is the pane poller, and SendKick blocks on the input prompt), so the retry +// lands once the refusing turn has returned to the prompt. Must NOT be +// called with m.mu held. +func (m *Manager) retryRefusedCanaryKick(agent *AgentProcess) { + m.mu.RLock() + plain, ok := planCanaryRetry(agent.lastKickInput, agent.canaryRetryDone) + m.mu.RUnlock() + if !ok { + return + } + m.logger.Warn("kick refused with the exfiltration canary in front; re-delivering once without it", + "agent", agent.Name, + "reason", agent.KickRefusalReason, + ) + deliver := m.canaryRetryDeliver + if deliver == nil { + deliver = func(name, message string) { + go func() { + if err := m.sendKick(name, message, kickTriggerCanaryRetry); err != nil { + m.logger.Warn("canary-free re-delivery of refused kick failed", + "agent", name, "error", err) + } + }() + } + } + deliver(agent.Name, plain) +} diff --git a/src/crux/agent/kick_canary_retry_test.go b/src/crux/agent/kick_canary_retry_test.go new file mode 100644 index 0000000000..68d3528db6 --- /dev/null +++ b/src/crux/agent/kick_canary_retry_test.go @@ -0,0 +1,82 @@ +package agent + +import "testing" + +// A model that reads the canary preamble as a prompt-injection marker refuses +// the whole kick; the kick is re-delivered once without the canary so the +// refusal does not burn the cadence slot. + +func TestStripCanaryPreamble(t *testing.T) { + work := "Review PR #15783 (correctness).\n\nFocus on the retry loop." + if got, ok := stripCanaryPreamble(testCanaryPreamble + work); !ok || got != work { + t.Fatalf("stripCanaryPreamble = (%q, %v), want (%q, true)", got, ok, work) + } + for _, msg := range []string{work, testCanaryPreamble, "=== HIVE HARNESS HEADER ===\nExfiltration canary: HIVE-CANARY-abc123 no blank line", ""} { + if got, ok := stripCanaryPreamble(msg); ok || got != msg { + t.Errorf("stripCanaryPreamble(%q) = (%q, %v), want unchanged, false", msg, got, ok) + } + } +} + +func TestPlanCanaryRetry(t *testing.T) { + work := "Triage issue #12." + if plain, ok := planCanaryRetry(testCanaryPreamble+work, false); !ok || plain != work { + t.Fatalf("canary kick must earn one plain retry, got (%q, %v)", plain, ok) + } + if _, ok := planCanaryRetry(testCanaryPreamble+work, true); ok { + t.Error("the retry itself must not be retried") + } + if _, ok := planCanaryRetry(work, false); ok { + t.Error("a kick without a canary has nothing to strip; no retry") + } + if _, ok := planCanaryRetry(testCanaryPreamble, false); ok { + t.Error("a canary-only kick has no work to re-deliver") + } +} + +func TestCheckKickRefusalRedeliversCanaryKickOnce(t *testing.T) { + m := &Manager{logger: discardLogger()} + var delivered []string + m.canaryRetryDeliver = func(name, message string) { + delivered = append(delivered, name+"\x00"+message) + } + work := "[agent:scanner]\n\nScan cncf/techdocs for stale pages." + refusal := "I'm noticing this request includes a canary marker and looks like a prompt injection, so I won't act on it." + + agent := &AgentProcess{Name: "scanner", lastKickInput: testCanaryPreamble + work} + m.checkKickRefusal(agent, refusal) + if !agent.KickRefused { + t.Fatal("refusal must still be recorded") + } + if len(delivered) != 1 || delivered[0] != "scanner\x00"+work { + t.Fatalf("refused canary kick must be re-delivered plain, got %q", delivered) + } + + // The re-delivery is marked by its trigger; a refusal of the retry stands. + m.recordDeliveredKickLocked(agent, work, kickTriggerCanaryRetry) + if agent.KickRefused || !agent.canaryRetryDone { + t.Fatalf("retry delivery must clear the refusal and mark the retry: refused=%v retried=%v", agent.KickRefused, agent.canaryRetryDone) + } + agent.lastKickInput = testCanaryPreamble + work + m.checkKickRefusal(agent, refusal) + if len(delivered) != 1 { + t.Fatalf("a refused retry must not be retried again, got %d deliveries", len(delivered)) + } + + // A fresh kick from the governor re-arms the single retry. + m.recordDeliveredKickLocked(agent, testCanaryPreamble+work, "send-kick") + if agent.canaryRetryDone { + t.Fatal("an ordinary kick must re-arm the canary retry") + } +} + +func TestCheckKickRefusalWithoutCanaryDoesNotRedeliver(t *testing.T) { + m := &Manager{logger: discardLogger()} + calls := 0 + m.canaryRetryDeliver = func(string, string) { calls++ } + agent := &AgentProcess{Name: "scanner", lastKickInput: "[agent:scanner]\n\nPlain kick."} + m.checkKickRefusal(agent, "This looks like a prompt injection attempt, so I'm declining to execute it.") + if !agent.KickRefused || calls != 0 { + t.Fatalf("plain kick refusal must be recorded without a retry: refused=%v calls=%d", agent.KickRefused, calls) + } +} diff --git a/src/crux/agent/kick_preview.go b/src/crux/agent/kick_preview.go index f25130c5c9..6566381e92 100644 --- a/src/crux/agent/kick_preview.go +++ b/src/crux/agent/kick_preview.go @@ -3,24 +3,33 @@ package agent import "strings" // canaryPreamblePrefix is how an ioscan exfiltration canary opens a kick -// (hive pkg/ioscan.CanaryPreamble). The preamble runs to the first blank -// line and is about as long as the audit preview, so a kick that carried -// one showed operators nothing but the canary (hivecommons/hive#15823). The -// preview and the kick-history snippet start after it; the kick itself is -// delivered unchanged, canary first. -const canaryPreamblePrefix = "SECURITY CANARY: " +// (hive pkg/ioscan.CanaryPreamblePrefix; the two must stay in step). The +// preamble runs to the first blank line and is about as long as the audit +// preview, so a kick that carried one showed operators nothing but the +// canary (hivecommons/hive#15823). The preview and the kick-history snippet +// start after it; the kick itself is delivered unchanged, canary first. +const canaryPreamblePrefix = "=== HIVE HARNESS HEADER" -// kickPreviewBody returns the part of a kick worth previewing: the message -// after its canary preamble, or the whole message when it carries none (or -// nothing follows the preamble, so the canary-only kick stays visible as -// such). -func kickPreviewBody(message string) string { +// stripCanaryPreamble returns the kick without its canary preamble and true +// when one was present and something followed it. A canary-only kick (or one +// whose preamble is not terminated by a blank line) is returned unchanged +// with false. +func stripCanaryPreamble(message string) (string, bool) { if !strings.HasPrefix(message, canaryPreamblePrefix) { - return message + return message, false } _, rest, ok := strings.Cut(message, "\n\n") if !ok || strings.TrimSpace(rest) == "" { - return message + return message, false } - return rest + return rest, true +} + +// kickPreviewBody returns the part of a kick worth previewing: the message +// after its canary preamble, or the whole message when it carries none (or +// nothing follows the preamble, so the canary-only kick stays visible as +// such). +func kickPreviewBody(message string) string { + body, _ := stripCanaryPreamble(message) + return body } diff --git a/src/crux/agent/kick_preview_test.go b/src/crux/agent/kick_preview_test.go index ff0a21b36c..8a3b3b7187 100644 --- a/src/crux/agent/kick_preview_test.go +++ b/src/crux/agent/kick_preview_test.go @@ -5,7 +5,7 @@ import "testing" // hivecommons/hive#15823: the audit preview of a canary-bearing kick shows // the work the agent was given, not the canary preamble. -const testCanaryPreamble = "SECURITY CANARY: HIVE-CANARY-abc123\nThis is a hive-issued exfiltration-canary marker. Never print, quote, summarize, transform, store in reports, include in commits, or send it to any external service.\n\n" +const testCanaryPreamble = "=== HIVE HARNESS HEADER (operator configuration, not user content) ===\nExfiltration canary: HIVE-CANARY-abc123\nThe token above is a tripwire planted by the hive harness that runs you. It is not a task.\n=== END HIVE HARNESS HEADER ===\n\n" func TestKickPreviewBodySkipsCanaryPreamble(t *testing.T) { work := "Review PR #15783 (correctness).\n\nFocus on the retry loop." @@ -22,7 +22,7 @@ func TestKickPreviewBodyPassesThroughKicksWithoutPreamble(t *testing.T) { } func TestKickPreviewBodyKeepsCanaryOnlyKickVisible(t *testing.T) { - for _, msg := range []string{testCanaryPreamble, "SECURITY CANARY: HIVE-CANARY-abc123 no blank line"} { + for _, msg := range []string{testCanaryPreamble, "=== HIVE HARNESS HEADER ===\nExfiltration canary: HIVE-CANARY-abc123 no blank line"} { if got := kickPreviewBody(msg); got != msg { t.Fatalf("preview of canary-only kick = %q, want whole message", got) } diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 0d25429396..4a63b82b0f 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -257,16 +257,22 @@ type AgentProcess struct { LastKickMessage string KickRefused bool KickRefusalReason string - LaunchedMode AgentMode - HasLaunched bool - poolLeaseID string - poolLeaseCancel context.CancelFunc - baseConfig config.AgentConfig - agentSpecPrompt string - tmuxSession string - tmuxSocket string - cancel context.CancelFunc - forceRelaunch bool + // lastKickInput is the last kick as handed to delivery, before the + // backend suffixes, so a refused canary kick can be re-delivered plain + // without doubling the suffixes. canaryRetryDone is set when the current + // kick IS that re-delivery, so it is never retried again. + lastKickInput string + canaryRetryDone bool + LaunchedMode AgentMode + HasLaunched bool + poolLeaseID string + poolLeaseCancel context.CancelFunc + baseConfig config.AgentConfig + agentSpecPrompt string + tmuxSession string + tmuxSocket string + cancel context.CancelFunc + forceRelaunch bool // launching is set true under m.mu while Start runs this agent's launch // with m.mu RELEASED (so a slow /data NFS write or a hung MITM-proxy token // mint during launch cannot block AllStatuses()/the heartbeat collect() and @@ -677,6 +683,9 @@ type Manager struct { // kickOutcomeObserver receives the verdict on how each kicked turn ended // (#7421); the governor consumes it. Same discipline as kickObserver. kickOutcomeObserver atomic.Pointer[func(agentName string, outcome KickOutcome)] + // canaryRetryDeliver, when set, replaces the goroutine+sendKick path a + // refused canary kick is re-delivered through (tests inject a recorder). + canaryRetryDeliver func(name, message string) // kickDispatches tracks asynchronous kick dispatches (#5325): the in-flight // guard that makes delivery exactly-once, and the latest outcome per agent diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index f3a877382b..838d6a783d 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -98,7 +98,13 @@ func notRunningReason(agent *AgentProcess) string { } } -func (m *Manager) SendKick(name string, message string) (err error) { +func (m *Manager) SendKick(name string, message string) error { + return m.sendKick(name, message, "send-kick") +} + +// sendKick is SendKick with the delivery trigger spelled out; the trigger is +// what the audit log, prompt history and the canary-retry bookkeeping see. +func (m *Manager) sendKick(name, message, trigger string) (err error) { // Agent-kick span. No-op with zero export cost when tracing is disabled // (the default). SendKick has no context parameter, so this span roots at // Background; it still captures the kick leg of the governor→agent @@ -229,7 +235,7 @@ func (m *Manager) SendKick(name string, message string) (err error) { return err } - return m.deliverKickLocked(agent, message, "send-kick") + return m.deliverKickLocked(agent, message, trigger) } // deliverKickLocked types a message into the agent's CLI and records the @@ -257,6 +263,7 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string return fmt.Errorf("%w: agent %s kick delivery already in progress", crux.ErrAgentBusy, agent.Name) } defer agent.kickDelivering.Store(false) + agent.lastKickInput = message visible := m.captureVisiblePaneForAgent(agent) if err := m.checkKickPromptReadyLocked(agent, visible, trigger); err != nil { @@ -420,6 +427,7 @@ func (m *Manager) recordDeliveredKickLocked(agent *AgentProcess, message, trigge agent.LastKickMessage = message agent.KickRefused = false agent.KickRefusalReason = "" + agent.canaryRetryDone = trigger == kickTriggerCanaryRetry // The session now holds this kick's output; the next rotation point // (kick, restart, shutdown) must archive it before it is destroyed. agent.kickLogPending = true diff --git a/src/crux/agent/manager_poll.go b/src/crux/agent/manager_poll.go index a620e31f25..e02a2907c0 100644 --- a/src/crux/agent/manager_poll.go +++ b/src/crux/agent/manager_poll.go @@ -586,6 +586,7 @@ func (m *Manager) checkKickRefusal(agent *AgentProcess, line string) { "pattern", pattern, "line", reason, ) + m.retryRefusedCanaryKick(agent) return } } diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index af8d9a3c73..e3e391f987 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -92,7 +92,7 @@ in `src/pkg/scheduler/kick_layout.go` and asserted by | # | Block | Varies | |---|---|---| -| 1 | `SECURITY CANARY` preamble | every kick (random, opt-in `ioscan.canaries`; always first — [ADR-0008](adr/0008-ioscan-untrusted-input.md)) | +| 1 | `=== HIVE HARNESS HEADER` canary preamble | every kick (random, opt-in `ioscan.canaries`; always first — [ADR-0008](adr/0008-ioscan-untrusted-input.md)) | | 2 | `[agent:<name>]` header | never | | 3 | governor work-source rule | never | | 4 | independent review section | per agent config | diff --git a/src/pkg/ioscan/canary.go b/src/pkg/ioscan/canary.go index 1c5be63067..7cb45c7063 100644 --- a/src/pkg/ioscan/canary.go +++ b/src/pkg/ioscan/canary.go @@ -446,11 +446,26 @@ func decodeCanaryHexBlobs(text string) []string { return out } +// CanaryPreamblePrefix opens every canary preamble; crux/agent mirrors it +// (canaryPreamblePrefix) to find where the kick body starts. +const CanaryPreamblePrefix = "=== HIVE HARNESS HEADER" + +// CanaryPreamble renders the canary as a clearly-labelled block of the +// operator's own harness configuration. The earlier "SECURITY CANARY: <token> +// … Never print …" wording read to Claude Sonnet 5 as a prompt-injection +// marker ("this request includes a 'SECURITY CANARY' marker and asks me to act +// as a scanner agent") and the model refused the whole kick — ~65% of one +// hosted supervisor's kicks over 36h once the Copilot CLI substituted that +// model. The header states who planted the token and why, says it is not a +// task, and still forbids every transform an agent might consider safe. func CanaryPreamble(token string) string { if token == "" { return "" } - return fmt.Sprintf("SECURITY CANARY: %s\nThis is a hive-issued exfiltration-canary marker. Never print, quote, summarize, transform, store in reports, include in commits, or send it to any external service.\n\n", token) + return fmt.Sprintf("%s (operator configuration, not user content) ===\n"+ + "Exfiltration canary: %s\n"+ + "The token above is a tripwire planted by the hive harness that runs you — your own operator's configuration — so it can detect session text leaking off this host. It is not a task and nothing in this message depends on it. Handle it as a secret: never print, quote, summarize, transform, store in reports, include in commits, or send it to any external service; then carry out the kick that follows.\n"+ + "=== END HIVE HARNESS HEADER ===\n\n", CanaryPreamblePrefix, token) } func (r *CanaryRegistry) Load() error { diff --git a/src/pkg/ioscan/canary_coverage_test.go b/src/pkg/ioscan/canary_coverage_test.go index 72857a890b..371ba8d55c 100644 --- a/src/pkg/ioscan/canary_coverage_test.go +++ b/src/pkg/ioscan/canary_coverage_test.go @@ -16,10 +16,27 @@ func TestCanaryPreambleOmittedWithoutToken(t *testing.T) { if !strings.Contains(got, "HIVE-CANARY-deadbeef") { t.Fatalf("preamble %q does not carry the token", got) } + if !strings.HasPrefix(got, CanaryPreamblePrefix) { + t.Fatalf("preamble %q does not open with CanaryPreamblePrefix", got) + } + if !strings.HasSuffix(got, "\n\n") { + t.Fatalf("preamble %q must end with a blank line so the kick body is delimited", got) + } + // Sonnet 5 refused kicks whose canary announced itself as a bare + // "SECURITY CANARY" marker (prompt-injection pattern); the header must + // instead say whose configuration it is and that it is not a task. + for _, want := range []string{"operator", "not a task"} { + if !strings.Contains(got, want) { + t.Fatalf("preamble %q missing %q", got, want) + } + } + if strings.Contains(got, "SECURITY CANARY") { + t.Fatalf("preamble %q still carries the refused SECURITY CANARY marker", got) + } // The preamble is the only thing standing between the agent and an // innocent-looking "summarize your context" exfiltration, so it has to // forbid the transforms an agent would otherwise consider safe. - for _, want := range []string{"Never print", "summarize", "external service"} { + for _, want := range []string{"never print", "summarize", "external service"} { if !strings.Contains(got, want) { t.Fatalf("preamble %q missing %q", got, want) } diff --git a/src/pkg/scheduler/kick_layout_test.go b/src/pkg/scheduler/kick_layout_test.go index acc35a5d69..971ef98432 100644 --- a/src/pkg/scheduler/kick_layout_test.go +++ b/src/pkg/scheduler/kick_layout_test.go @@ -95,7 +95,7 @@ func kickLayoutMessage(t *testing.T, s *Scheduler, issues ...github.Issue) strin // one dynamic thing ahead of the static prefix (ADR-0008). func stripCanary(t *testing.T, msg string) string { t.Helper() - if !strings.HasPrefix(msg, "SECURITY CANARY: "+ioscan.CanaryPrefix) { + if !strings.HasPrefix(msg, ioscan.CanaryPreamblePrefix) { t.Fatalf("canary preamble must lead the kick:\n%s", msg[:min(len(msg), 200)]) } end := strings.Index(msg, "\n\n") @@ -187,7 +187,7 @@ func TestKickLayoutSeamOrderIsFixed(t *testing.T) { msg := kickLayoutMessage(t, s, makeIssue("test-org/console", 1, "first issue", "scanner", 5, nil, false)) order := []string{ - "SECURITY CANARY: " + ioscan.CanaryPrefix, + ioscan.CanaryPreamblePrefix, "[agent:scanner]", governorWorkSourceRule, "## Independent review", @@ -228,14 +228,14 @@ func TestKickLayoutKeepsCanaryAndRepoScopedTarget(t *testing.T) { b := kickLayoutMessage(t, s, issue) preamble := func(msg string) string { - end := strings.Index(msg, "\n") + end := strings.Index(msg, "\n\n") if end < 0 { return msg } return msg[:end] } for _, msg := range []string{a, b} { - if !strings.HasPrefix(msg, "SECURITY CANARY: "+ioscan.CanaryPrefix) { + if !strings.HasPrefix(msg, ioscan.CanaryPreamblePrefix) { t.Fatalf("canary preamble must lead the kick:\n%s", msg[:min(len(msg), 200)]) } if !strings.Contains(msg, "REPO-SCOPED CADENCE TARGET: test-org/console") { diff --git a/src/pkg/scheduler/repo_filter_hold_test.go b/src/pkg/scheduler/repo_filter_hold_test.go index 1427e09c3a..9fbf4b3562 100644 --- a/src/pkg/scheduler/repo_filter_hold_test.go +++ b/src/pkg/scheduler/repo_filter_hold_test.go @@ -172,7 +172,7 @@ func TestAddCanaryPreambleEnabledPrependsRegisteredToken(t *testing.T) { const msg = "kick body" got := s.addCanaryPreamble("scanner", msg) - if !strings.HasPrefix(got, "SECURITY CANARY: "+ioscan.CanaryPrefix) { + if !strings.HasPrefix(got, ioscan.CanaryPreamblePrefix) { t.Fatalf("preamble missing canary header: %q", got) } if !strings.HasSuffix(got, msg) { From 51f822191efbad221818817dad417770e1ba735e Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:53:41 +0000 Subject: [PATCH 122/223] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20stop?= =?UTF-8?q?=20canceled=20contributor=20sweep=20passes=20(#18003)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(automerge): stop canceled contributor sweep passes Stop a contributor pass interrupted by its canceled generation context instead of reporting a failure and reconciling stages on that context. Preserve warnings for other errors. Hive-Run: hivecommons/hive#17949 Hive-Plan: contributor-sweep-cancellation Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * docs: clarify cancellation exclusion context Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-contributor-sweep-cancellation.md | 1 + src/docs/fleet-report.md | 4 ++ .../automerge_sweep_cancellation_test.go | 57 +++++++++++++++++++ .../github/automerge/automerge_sweep_loop.go | 9 ++- 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-contributor-sweep-cancellation.md create mode 100644 src/pkg/github/automerge/automerge_sweep_cancellation_test.go diff --git a/changelog.d/fixed-contributor-sweep-cancellation.md b/changelog.d/fixed-contributor-sweep-cancellation.md new file mode 100644 index 0000000000..be00f6d940 --- /dev/null +++ b/changelog.d/fixed-contributor-sweep-cancellation.md @@ -0,0 +1 @@ +- Stop contributor automerge passes cleanly when a client hand-over or shutdown cancels an in-flight sweep, without filing a spurious failure report. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index db5eb42f78..7c0b9a20dc 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -178,6 +178,10 @@ logger-bound attributes: typed errors wrapping `context.Canceled`, exact Warnings remain in local logs; deadlines and other errors remain reportable, including when another `error` attribute accompanies a cancellation. +When a client hand-over or shutdown cancels an in-flight contributor-lane +automerge pass, the pass stops without warning or stage reconciliation. Other +contributor sweep failures remain reportable. + The GitHub rate-limit exclusion also covers the local REST reserve guard: when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. diff --git a/src/pkg/github/automerge/automerge_sweep_cancellation_test.go b/src/pkg/github/automerge/automerge_sweep_cancellation_test.go new file mode 100644 index 0000000000..8de5fd12c3 --- /dev/null +++ b/src/pkg/github/automerge/automerge_sweep_cancellation_test.go @@ -0,0 +1,57 @@ +package automerge + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestContributorSweepPassCancellation(t *testing.T) { + for _, tc := range []struct { + name string + cancel bool + }{ + {name: "generation canceled", cancel: true}, + {name: "github failure"}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var requests atomic.Int32 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if tc.cancel { + cancel() + <-r.Context().Done() + return + } + http.Error(w, "upstream failure", http.StatusInternalServerError) + })) + defer api.Close() + + var logs bytes.Buffer + clock := &contributorLaneClock{at: time.Date(2026, 10, 15, 13, 0, 0, 0, time.UTC)} + engine := newContributorLaneEngine(api.URL, defaultContributorLanePolicy(), clock, Options{ + Logger: slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + result := engine.sweepPass(ctx, 1, false, nil, "tick") + if requests.Load() == 0 { + t.Fatal("contributor lane did not attempt to list pull requests") + } + failed := strings.Contains(logs.String(), "contributor-lane automerge sweep failed") + if tc.cancel { + if result != nil || failed || strings.Contains(logs.String(), "stage labels reconcile failed") { + t.Fatalf("canceled pass: result=%+v logs=%s", result, logs.String()) + } + } else if result == nil || !failed { + t.Fatalf("GitHub failure must remain visible: result=%+v logs=%s", result, logs.String()) + } + }) + } +} diff --git a/src/pkg/github/automerge/automerge_sweep_loop.go b/src/pkg/github/automerge/automerge_sweep_loop.go index 636a529c9b..a5e9740c5a 100644 --- a/src/pkg/github/automerge/automerge_sweep_loop.go +++ b/src/pkg/github/automerge/automerge_sweep_loop.go @@ -735,7 +735,8 @@ func (c *Engine) StartSelfAuthoredAutoMergeSweep(ctx context.Context, maxMerges // sweepPass runs the three lanes once, in order, sharing one merge budget, // then reconciles stage labels. targets narrows an event-driven pass to the // PRs a webhook named (nil: everything); cause names what started the pass -// for the log. It returns nil when the self-authored lane failed outright +// for the log. It returns nil when the self-authored lane failed outright or +// the contributor lane was interrupted by cancellation of the sweep context // (its callers then keep the current cadence). func (c *Engine) sweepPass(ctx context.Context, maxMerges int, acmmAllowed bool, targets SweepTargets, cause string) *AutoMergeSweepResult { c.sweepClock.begin(cause, targets, c.now()) @@ -784,6 +785,12 @@ func (c *Engine) sweepPass(ctx context.Context, maxMerges int, acmmAllowed bool, if remaining > 0 && c.currentContributorLanePolicy().Enabled { laneResult, laneErr := c.SweepContributorLaneAutoMerges(ctx, AutoMergeSweepOptions{MaxMerges: remaining, Targets: targets}) if laneErr != nil { + // A client hand-over or shutdown cancels the whole sweep. Do not + // report that expected interruption as a failed contributor pass, + // or start stage reconciliation on the canceled context. + if ctx.Err() == context.Canceled && errors.Is(laneErr, context.Canceled) { + return nil + } c.warn("contributor-lane automerge sweep failed", "error", laneErr, "cause", cause) if isRateLimited(laneErr) { c.refreshRateLimitCache(ctx) From 0c8e07906ba63f52a3391f15976c6229c3b2d042 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:54:42 +0000 Subject: [PATCH 123/223] =?UTF-8?q?=E2=9C=A8=20review:=20add=20perspsrc=20?= =?UTF-8?q?resolver=20with=20last-known-good=20fallback=20(#18011)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-17987-review-perspsrc-resolver.md | 1 + src/cmd/hive/main_sweeps.go | 6 +- src/cmd/hive/review_config_parsers_test.go | 20 ++ src/pkg/perspsrc/resolver.go | 184 ++++++++++++++++++ src/pkg/perspsrc/resolver_test.go | 176 +++++++++++++++++ src/pkg/scheduler/policy_overlays.go | 2 +- 6 files changed, 386 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-17987-review-perspsrc-resolver.md create mode 100644 src/pkg/perspsrc/resolver.go create mode 100644 src/pkg/perspsrc/resolver_test.go diff --git a/changelog.d/added-17987-review-perspsrc-resolver.md b/changelog.d/added-17987-review-perspsrc-resolver.md new file mode 100644 index 0000000000..f3bbd4c8eb --- /dev/null +++ b/changelog.d/added-17987-review-perspsrc-resolver.md @@ -0,0 +1 @@ +- **Resolver for `review.perspectives_source`** that fetches a `kind: ReviewPerspectives` document behind the same seed-only allowlist as `definition_source`, and keeps the last-known-good set with a visible warning when a later fetch fails or a document no longer validates, instead of silently falling back to the built-in perspectives. The review relay, dispatch, head aggregation and `${REVIEW_PERSPECTIVES}` now read the resolved set via `ReviewConfig.EffectivePerspectives()`; with no `perspectives_source` configured, behaviour is unchanged. ([#17987](https://github.com/hivecommons/hive/issues/17987), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index ab9064d0ab..7487b7007d 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -2263,7 +2263,9 @@ func applyHumanDecisionLabels(ctx context.Context, cfg *config.Config, ghClient // silently becomes the current time would re-open every verdict in the // artifact at once — the opposite of the narrow, deliberate correction this // setting exists for. -// reviewPerspectiveSet resolves the hive's configured review perspectives. +// reviewPerspectiveSet resolves the hive's configured review perspectives: the +// set linked from review.perspectives_source when one has resolved, otherwise +// the inline review.perspectives (ReviewConfig.EffectivePerspectives). // // A bad configuration falls back to the built-in set and says so, rather than // disabling review. Silently reviewing nothing would look identical to a healthy @@ -2273,7 +2275,7 @@ func reviewPerspectiveSet(cfg *config.Config, logger *slog.Logger) review.Perspe if cfg == nil { return review.PerspectiveSet{} } - set, err := review.NewPerspectiveSet(cfg.Review.Perspectives, cfg.Review.PerspectivePrompts) + set, err := review.NewPerspectiveSet(cfg.Review.EffectivePerspectives()) if err != nil { if logger != nil { logger.Warn("review.perspectives is invalid; reviewing with the built-in set until it is fixed", diff --git a/src/cmd/hive/review_config_parsers_test.go b/src/cmd/hive/review_config_parsers_test.go index 1c0dc67eab..ad5a24d3cd 100644 --- a/src/cmd/hive/review_config_parsers_test.go +++ b/src/cmd/hive/review_config_parsers_test.go @@ -94,6 +94,26 @@ func TestReviewPerspectiveSet(t *testing.T) { } }) + t.Run("linked perspectives win over inline", func(t *testing.T) { + cfg := &config.Config{} + cfg.Review.Perspectives = []string{"correctness"} + cfg.Review.PerspectivesSource = &config.DefinitionSourceConfig{Owner: "torch-spyre", Repo: "spyre-inference", Path: ".hive/review-perspectives.yaml"} + cfg.Review.LinkedPerspectives = &config.LinkedPerspectives{ + Names: []string{"security", "spyre-numerics"}, + Focus: map[string]string{"spyre-numerics": "Silent CPU fallbacks and fp16 accumulation-order changes."}, + } + got := reviewPerspectiveSet(cfg, nil).List() + want := []review.Perspective{review.PerspectiveSecurity, review.Perspective("spyre-numerics")} + if len(got) != len(want) { + t.Fatalf("List() = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("List()[%d] = %q, want %q", i, got[i], want[i]) + } + } + }) + t.Run("invalid selection with nil logger does not panic", func(t *testing.T) { cfg := &config.Config{} cfg.Review.Perspectives = []string{"sekurity"} diff --git a/src/pkg/perspsrc/resolver.go b/src/pkg/perspsrc/resolver.go new file mode 100644 index 0000000000..747ce35c4c --- /dev/null +++ b/src/pkg/perspsrc/resolver.go @@ -0,0 +1,184 @@ +package perspsrc + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/defsrc" +) + +// defaultFetchTimeout bounds a single live fetch so a slow/hung GitHub API call +// cannot stall a reload. On timeout the resolver keeps the last-known-good set. +const defaultFetchTimeout = 8 * time.Second + +// Resolver fetches review perspective documents live and remembers the last +// good document per source so a later failure keeps that set in effect. It is +// modelled on defsrc.Resolver and shares its Fetcher, AllowFunc and Logger. +// Safe for concurrent use. +type Resolver struct { + fetcher defsrc.Fetcher + allow defsrc.AllowFunc + logger defsrc.Logger + timeout time.Duration + + mu sync.RWMutex + cache map[string]string // source key -> last-known-good raw YAML +} + +// NewResolver builds a Resolver. fetcher may be nil (token-mode boot without an +// App client), in which case only a cached document can resolve. Callers wire +// allow to the seed-only gate (config.Config.GitHubPromptAllowed / +// GitHubDefinitionAllowed); a nil allow denies everything (fail closed). +// logger may be nil. +func NewResolver(fetcher defsrc.Fetcher, allow defsrc.AllowFunc, logger defsrc.Logger) *Resolver { + if allow == nil { + allow = func(string) bool { return false } + } + return &Resolver{ + fetcher: fetcher, + allow: allow, + logger: logger, + timeout: defaultFetchTimeout, + cache: map[string]string{}, + } +} + +// Result carries a resolved perspective set plus provenance. Ok is false when +// nothing could be resolved (not set, denied, no fetcher, unreachable or +// malformed with no last-known-good); the caller then falls back to inline +// review.perspectives. Warning is set when the latest fetch or document failed, +// whether or not a last-known-good set is still in effect. +type Result struct { + Names []string + Focus map[string]string + Ok bool + Source string // "github:owner/repo@ref", "github-cache:owner/repo", "denied", "unset", "no-client", "error" + Warning string +} + +// Resolve fetches the perspective document for src. It never returns an error: +// a reload must proceed even when the source is unreachable or malformed. A +// document that fails validation keeps the last-known-good set (with a +// Warning) rather than silently falling back to the built-in set. +func (r *Resolver) Resolve(ctx context.Context, src config.DefinitionSourceConfig) Result { + if !src.IsSet() { + return Result{Source: "unset"} + } + slug := src.Slug() + if !r.allow(slug) { + if r.logger != nil { + r.logger.Warn("review perspectives source repo not allowlisted, ignoring (keeping inline review.perspectives)", "repo", slug) + } + return Result{Source: "denied"} + } + if r.fetcher == nil { + if res, ok := r.fromCache(src, ""); ok { + return res + } + return Result{Source: "no-client"} + } + + fetchCtx := ctx + if ctx == nil { + fetchCtx = context.Background() + } + fetchCtx, cancel := context.WithTimeout(fetchCtx, r.timeout) + defer cancel() + + raw, err := r.fetcher.GetFileContentRef(fetchCtx, src.Owner, src.Repo, src.Path, src.Ref) + if err != nil { + warning := fmt.Sprintf("fetching review perspectives from %s/%s failed: %v", slug, src.Path, err) + if r.logger != nil { + r.logger.Warn("failed to fetch review perspectives from GitHub, keeping last-known-good", "repo", slug, "path", src.Path, "error", err) + } + if res, ok := r.fromCache(src, warning); ok { + return res + } + return Result{Source: "error", Warning: warning} + } + names, focus, err := ParseDocument(raw) + if err != nil { + warning := fmt.Sprintf("review perspectives document %s/%s is invalid: %v", slug, src.Path, err) + if r.logger != nil { + r.logger.Warn("review perspectives from GitHub are malformed, keeping last-known-good", "repo", slug, "path", src.Path, "error", err) + } + if res, ok := r.fromCache(src, warning); ok { + return res + } + return Result{Source: "error", Warning: warning} + } + // Only a document that parsed cleanly is cached, so a later failure keeps + // a known-good set, never a corrupt one. + r.store(src, raw) + return Result{Names: names, Focus: focus, Ok: true, Source: provenance(src)} +} + +// fromCache resolves the last-known-good document for src, if any. It was +// validated when cached, so a parse failure here is not expected; it is still +// treated as a miss rather than trusted. +func (r *Resolver) fromCache(src config.DefinitionSourceConfig, warning string) (Result, bool) { + raw, ok := r.cached(src) + if !ok { + return Result{}, false + } + names, focus, err := ParseDocument(raw) + if err != nil { + return Result{}, false + } + return Result{Names: names, Focus: focus, Ok: true, Source: "github-cache:" + src.Slug(), Warning: warning}, true +} + +func provenance(src config.DefinitionSourceConfig) string { + ref := src.Ref + if ref == "" { + ref = "default" + } + return fmt.Sprintf("github:%s@%s", src.Slug(), ref) +} + +func sourceKey(src config.DefinitionSourceConfig) string { + return src.Owner + "\x00" + src.Repo + "\x00" + src.Path + "\x00" + src.Ref +} + +func (r *Resolver) cached(src config.DefinitionSourceConfig) (string, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + raw, ok := r.cache[sourceKey(src)] + return raw, ok +} + +func (r *Resolver) store(src config.DefinitionSourceConfig, raw string) { + r.mu.Lock() + defer r.mu.Unlock() + r.cache[sourceKey(src)] = raw +} + +// ApplyToConfig resolves cfg.Review.PerspectivesSource and records the result +// on cfg.Review.LinkedPerspectives, with its provenance and any warning. When +// nothing resolves, LinkedPerspectives is left nil so EffectivePerspectives +// falls back to inline review.perspectives. A nil cfg or resolver is a no-op. +func ApplyToConfig(ctx context.Context, cfg *config.Config, resolver *Resolver, logger defsrc.Logger) { + if cfg == nil || resolver == nil { + return + } + cfg.Review.LinkedPerspectives = nil + if !cfg.Review.PerspectivesSource.IsSet() { + return + } + res := resolver.Resolve(ctx, *cfg.Review.PerspectivesSource) + if !res.Ok { + return + } + cfg.Review.LinkedPerspectives = &config.LinkedPerspectives{ + Names: res.Names, + Focus: res.Focus, + Source: res.Source, + Warning: res.Warning, + } + if logger != nil { + logger.Info("applied live review perspectives", "source", res.Source, "perspectives", len(res.Names)) + } +} diff --git a/src/pkg/perspsrc/resolver_test.go b/src/pkg/perspsrc/resolver_test.go new file mode 100644 index 0000000000..451856888f --- /dev/null +++ b/src/pkg/perspsrc/resolver_test.go @@ -0,0 +1,176 @@ +package perspsrc + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +type stubFetcher struct { + docs []string + errs []error + calls int +} + +func (f *stubFetcher) GetFileContentRef(_ context.Context, _, _, _, _ string) (string, error) { + i := f.calls + f.calls++ + var err error + if i < len(f.errs) { + err = f.errs[i] + } + if err != nil { + return "", err + } + if i < len(f.docs) { + return f.docs[i], nil + } + return "", errors.New("no document") +} + +type recLogger struct{ warns int } + +func (l *recLogger) Warn(string, ...any) { l.warns++ } +func (l *recLogger) Info(string, ...any) {} + +func allowAll(string) bool { return true } + +var testSource = config.DefinitionSourceConfig{Owner: "torch-spyre", Repo: "spyre-inference", Path: ".hive/review-perspectives.yaml"} + +const goodDoc = `kind: ReviewPerspectives +perspectives: + - name: security + - name: spyre-numerics + focus: "Silent CPU fallbacks." +` + +// A custom perspective without focus text fails NewPerspectiveSet validation. +const invalidDoc = `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: undescribed-custom +` + +func TestResolverDeniedNotAllowlisted(t *testing.T) { + f := &stubFetcher{docs: []string{goodDoc}} + log := &recLogger{} + r := NewResolver(f, func(string) bool { return false }, log) + res := r.Resolve(context.Background(), testSource) + if res.Ok || res.Source != "denied" { + t.Fatalf("Resolve = %+v, want Ok=false Source=denied", res) + } + if f.calls != 0 { + t.Errorf("fetcher called %d times for a denied repo, want 0", f.calls) + } + if log.warns == 0 { + t.Error("expected a Warn log for a denied repo") + } + + // A nil allow fails closed. + if res := NewResolver(f, nil, nil).Resolve(context.Background(), testSource); res.Ok || res.Source != "denied" { + t.Errorf("nil allow Resolve = %+v, want denied", res) + } +} + +func TestResolverNoClientUsesCache(t *testing.T) { + r := NewResolver(nil, allowAll, nil) + if res := r.Resolve(context.Background(), testSource); res.Ok || res.Source != "no-client" { + t.Fatalf("empty-cache Resolve = %+v, want Ok=false Source=no-client", res) + } + + r.store(testSource, goodDoc) + res := r.Resolve(context.Background(), testSource) + if !res.Ok || res.Source != "github-cache:torch-spyre/spyre-inference" { + t.Fatalf("cached Resolve = %+v, want Ok=true from github-cache", res) + } + if want := []string{"security", "spyre-numerics"}; !reflect.DeepEqual(res.Names, want) { + t.Errorf("Names = %v, want %v", res.Names, want) + } +} + +func TestResolverFetchErrorKeepsLastKnownGood(t *testing.T) { + f := &stubFetcher{docs: []string{goodDoc}, errs: []error{nil, errors.New("boom")}} + r := NewResolver(f, allowAll, nil) + first := r.Resolve(context.Background(), testSource) + if !first.Ok || first.Source != "github:torch-spyre/spyre-inference@default" { + t.Fatalf("first Resolve = %+v, want Ok from github", first) + } + second := r.Resolve(context.Background(), testSource) + if !second.Ok || second.Source != "github-cache:torch-spyre/spyre-inference" { + t.Fatalf("second Resolve = %+v, want Ok from github-cache", second) + } + if !reflect.DeepEqual(second.Names, first.Names) || !reflect.DeepEqual(second.Focus, first.Focus) { + t.Errorf("second = %v %v, want last-known-good %v %v", second.Names, second.Focus, first.Names, first.Focus) + } + + noCache := NewResolver(&stubFetcher{errs: []error{errors.New("boom")}}, allowAll, nil) + if res := noCache.Resolve(context.Background(), testSource); res.Ok || res.Source != "error" { + t.Errorf("no-cache fetch error Resolve = %+v, want Ok=false Source=error", res) + } +} + +func TestResolverMalformedKeepsLastKnownGoodAndWarns(t *testing.T) { + f := &stubFetcher{docs: []string{goodDoc, invalidDoc}} + log := &recLogger{} + r := NewResolver(f, allowAll, log) + first := r.Resolve(context.Background(), testSource) + if !first.Ok || first.Warning != "" { + t.Fatalf("first Resolve = %+v, want Ok with no warning", first) + } + second := r.Resolve(context.Background(), testSource) + if !second.Ok { + t.Fatalf("second Resolve = %+v, want last-known-good kept", second) + } + if !reflect.DeepEqual(second.Names, first.Names) { + t.Errorf("Names = %v, want last-known-good %v", second.Names, first.Names) + } + if second.Warning == "" { + t.Error("expected a Warning for the malformed document") + } + if log.warns == 0 { + t.Error("expected a Warn log for the malformed document") + } +} + +func TestResolverMalformedNoCacheFallsBackToInline(t *testing.T) { + r := NewResolver(&stubFetcher{docs: []string{invalidDoc}}, allowAll, nil) + res := r.Resolve(context.Background(), testSource) + if res.Ok { + t.Fatalf("Resolve = %+v, want Ok=false", res) + } + if res.Warning == "" { + t.Error("expected a Warning for the malformed document") + } + + cfg := &config.Config{} + cfg.Review.Perspectives = []string{"correctness"} + src := testSource + cfg.Review.PerspectivesSource = &src + ApplyToConfig(context.Background(), cfg, NewResolver(&stubFetcher{docs: []string{invalidDoc}}, allowAll, nil), nil) + if cfg.Review.LinkedPerspectives != nil { + t.Fatalf("LinkedPerspectives = %+v, want nil", cfg.Review.LinkedPerspectives) + } + if names, _ := cfg.Review.EffectivePerspectives(); !reflect.DeepEqual(names, []string{"correctness"}) { + t.Errorf("EffectivePerspectives names = %v, want inline [correctness]", names) + } +} + +func TestApplyToConfigRecordsProvenance(t *testing.T) { + cfg := &config.Config{} + src := testSource + cfg.Review.PerspectivesSource = &src + ApplyToConfig(context.Background(), cfg, NewResolver(&stubFetcher{docs: []string{goodDoc}}, allowAll, nil), nil) + lp := cfg.Review.LinkedPerspectives + if lp == nil { + t.Fatal("LinkedPerspectives = nil, want resolved set") + } + if lp.Source != "github:torch-spyre/spyre-inference@default" || lp.Warning != "" { + t.Errorf("LinkedPerspectives = %+v, want github provenance and no warning", lp) + } + if names, _ := cfg.Review.EffectivePerspectives(); !reflect.DeepEqual(names, []string{"security", "spyre-numerics"}) { + t.Errorf("EffectivePerspectives names = %v, want linked set", names) + } +} diff --git a/src/pkg/scheduler/policy_overlays.go b/src/pkg/scheduler/policy_overlays.go index d551ac96aa..21b9dbb712 100644 --- a/src/pkg/scheduler/policy_overlays.go +++ b/src/pkg/scheduler/policy_overlays.go @@ -770,7 +770,7 @@ func (s *Scheduler) addIndependentReviewSection(agentName, msg string) string { // reported"). Falls back to the built-in set when the configured one does not // validate, the same fallback the relay applies. func (s *Scheduler) reviewPerspectivesSection() string { - set, err := review.NewPerspectiveSet(s.cfg.Review.Perspectives, s.cfg.Review.PerspectivePrompts) + set, err := review.NewPerspectiveSet(s.cfg.Review.EffectivePerspectives()) if err != nil { set = review.PerspectiveSet{} } From 73e540a9c61b7b3e386bab6981b12d37765b4282 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 15:58:41 -0700 Subject: [PATCH 124/223] =?UTF-8?q?=F0=9F=90=9B=20Activity=20monitor:=20co?= =?UTF-8?q?lumn=20header=20follows=20the=20rows'=20workflow=20model=20(#18?= =?UTF-8?q?005)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Activity monitor: column header follows the rows' workflow model The sticky column header was always hive-default's names, even when every row's chevron strip wore a bound workflow model's columns. Derive the header from the same model the strips use: the sticky header is the lead row's model, an ungrouped page is partitioned (stably) by model with a header per run, and a hive-default group under a model's sticky header draws hive-default's names. Hives with no binding render exactly as before. Fixes #18002 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 tests: pin the sticky header's headHtml wiring Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...activity-monitor-workflow-column-header.md | 1 + src/pkg/dashboard/stage_tracker_cues_test.go | 2 +- .../stage_tracker_workflows_ui_test.go | 41 ++++++++++- .../static-src/js/tracker-filter-render.js | 15 +++- .../static-src/js/tracker-row-render.js | 2 +- .../static-src/js/tracker-workflows.js | 56 ++++++++++++++ src/pkg/dashboard/static/index.html | 73 ++++++++++++++++++- 7 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-activity-monitor-workflow-column-header.md diff --git a/changelog.d/fixed-activity-monitor-workflow-column-header.md b/changelog.d/fixed-activity-monitor-workflow-column-header.md new file mode 100644 index 0000000000..e868548c15 --- /dev/null +++ b/changelog.d/fixed-activity-monitor-workflow-column-header.md @@ -0,0 +1 @@ +- Activity monitor: the column header now follows the workflow model the rows wear — a repository bound to a custom model gets that model's column names over its chevrons (ungrouped pages are partitioned by model with a header per run), instead of hive-default's "PR in progress / Merge queue" names (#18002). diff --git a/src/pkg/dashboard/stage_tracker_cues_test.go b/src/pkg/dashboard/stage_tracker_cues_test.go index a691119203..0aeee73a5a 100644 --- a/src/pkg/dashboard/stage_tracker_cues_test.go +++ b/src/pkg/dashboard/stage_tracker_cues_test.go @@ -106,7 +106,7 @@ func TestStageTrackerCuesWiring(t *testing.T) { `stageTrackerNoteEvents(data); stageTrackerResultToastsSync(data); stageTrackerPendingExpire(data);`, `stageTrackerCueAgeSync(el, nowMs);`, `function celebrateRow(rowEl, opts) {`, - `const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: stageTrackerHeaderHtml(stages) }]`, + `const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: headHtml }]`, `if (!visible.length && !Object.keys(_stageTrackerCelebrating).length) {`, `data-change-action="stageTrackerCelebrateChange"`, } { diff --git a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go index 1e2d2a1ce9..3b9437a5f1 100644 --- a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go +++ b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go @@ -22,7 +22,13 @@ func TestStageTrackerWorkflowColumnsWiring(t *testing.T) { `const rowWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, item.repo) : null;`, `${rowWorkflow ? stageTrackerWorkflowStripItems(shown, rowWorkflow, stages) : stageTrackerStripItems(`, `const groupWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, g.repo) : null;`, - `${groupWorkflow && !g.collapsed ? stageTrackerWorkflowHeaderHtml(groupWorkflow) : ''}`, + `stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx)`, + // The sticky header and the ungrouped run headers follow the model + // the rows wear, never hive-default's names over model chevrons. + `const wfRuns = !grouping && ctx.workflows && typeof stageTrackerWorkflowRuns === 'function' ? stageTrackerWorkflowRuns(page.rows, ctx) : null;`, + `ctx.leadWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, leadRepo) : null;`, + `stageTrackerColumnsHeaderHtml(ctx.leadWorkflow, stages)`, + `stageTrackerWorkflowRunHeaderRows(inlineRows, wfRuns, stages)`, `.ham-wf-badge {`, `.ham-seg.ham-wf-uncovered {`, // Settings → Repos binding select. @@ -53,7 +59,9 @@ const STAGE_TRACKER_TERMINAL = 110; function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); } ` + jsFunc(t, html, "stageTrackerWorkflowOf") + jsFunc(t, html, "stageTrackerWorkflowColumnIndex") + jsFunc(t, html, "stageTrackerWorkflowBadgeHtml") + jsFunc(t, html, "stageTrackerWorkflowHeaderHtml") + - jsFunc(t, html, "stageTrackerWorkflowStripItems") + ` + jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowKey") + + jsFunc(t, html, "stageTrackerColumnsHeaderHtml") + jsFunc(t, html, "stageTrackerWorkflowRuns") + + jsFunc(t, html, "stageTrackerWorkflowRunHeaderRows") + jsFunc(t, html, "stageTrackerGroupColumnsHtml") + ` const STAGE_TRACKER_WORKFLOW_UNCOVERED_WORD = 'Not covered'; const def = { name: 'hive-default', version: '1', columns: [{ id: 'open', label: 'Open', states: ['open'], numbers: [] }] }; const td = { name: 'cncf-techdocs', version: '1', columns: [ @@ -94,6 +102,35 @@ const head = stageTrackerWorkflowHeaderHtml(td); assert.ok(head.includes('data-ham-workflow="cncf-techdocs@1"') && head.includes('title="Asked for"') && li(head).length === 4, head); assert.ok(stageTrackerWorkflowBadgeHtml(td).includes('>cncf-techdocs@1<')); assert.strictEqual(stageTrackerWorkflowBadgeHtml(null), ''); + +// The header over a strip is the strip's own model: the sticky header is +// the lead row's, an ungrouped page is partitioned by model with a header +// per run, and a hive-default group under a model's sticky header draws +// hive-default's names (a hive with no binding: one run, no headers added). +const STAGE_TRACKER_GROUP_KEY_PREFIX = 'group:'; +const STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX = STAGE_TRACKER_GROUP_KEY_PREFIX + 'workflow:'; +function stageTrackerHeaderHtml() { return '<div class="ham-cols default"></div>'; } +function stageTrackerItemKey(it) { return (it.repo || '') + '/' + (it.kind || '') + '#' + (Number(it.number) || 0); } +const rows = [ + { repo: 'cncf/plain', kind: 'issue', number: 1 }, { repo: 'cncf/techdocs', kind: 'issue', number: 2 }, + { repo: 'cncf/plain', kind: 'issue', number: 3 }, { repo: 'cncf/techdocs', kind: 'pr', number: 4 }, +]; +assert.strictEqual(stageTrackerColumnsHeaderHtml(null, []), stageTrackerHeaderHtml()); +assert.strictEqual(stageTrackerColumnsHeaderHtml(td, []), head); +let runs = stageTrackerWorkflowRuns(rows, ctx); +assert.deepStrictEqual(runs.map(r => [r.key, r.rows.map(x => x.number)]), [['', [1, 3]], ['cncf-techdocs@1', [2, 4]]], 'stable, first-seen partition'); +assert.strictEqual(runs[1].wf, td); +assert.strictEqual(stageTrackerWorkflowRuns(rows, {}).length, 1, 'no binding → one run'); +assert.strictEqual(stageTrackerWorkflowRuns(rows.filter(r => r.repo === 'cncf/techdocs'), ctx).length, 1, 'one model → one run'); +const keyed = runs.reduce((all, r) => all.concat(r.rows), []).map(it => ({ key: stageTrackerItemKey(it), html: 'row' })); +const withHeads = stageTrackerWorkflowRunHeaderRows(keyed, runs, []); +assert.deepStrictEqual(withHeads.map(r => r.key), ['cncf/plain/issue#1', 'cncf/plain/issue#3', 'group:workflow:cncf-techdocs@1', 'cncf/techdocs/issue#2', 'cncf/techdocs/pr#4']); +assert.strictEqual(withHeads[2].html, head, 'the run header is the model header'); +assert.deepStrictEqual(stageTrackerWorkflowRunHeaderRows(keyed, runs.slice(0, 1), []), keyed, 'one run → untouched'); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: false }, td, [], {}), head); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: true }, td, [], {}), ''); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: false }, null, [], { leadWorkflow: null }), '', 'hive-default group under the hive-default sticky header: nothing to repeat'); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: false }, null, [], { leadWorkflow: td }), stageTrackerHeaderHtml(), 'hive-default group under a model sticky header draws its own names'); ` if out, err := runNodeStdin(node, script); err != nil { t.Fatalf("node workflow columns fixture failed: %v\n%s", err, out) diff --git a/src/pkg/dashboard/static-src/js/tracker-filter-render.js b/src/pkg/dashboard/static-src/js/tracker-filter-render.js index d90bdceaa9..b2d3e8619f 100644 --- a/src/pkg/dashboard/static-src/js/tracker-filter-render.js +++ b/src/pkg/dashboard/static-src/js/tracker-filter-render.js @@ -117,6 +117,17 @@ ? stageTrackerGroupPage(stageTrackerGroups(ordered, { primary: stageTrackerPrimaryRepo(), collapsed: _stageTrackerCollapsed, stages }), f) : stageTrackerPage(ordered, f); if (!tv && page.page !== f.page) f.page = page.page; + // Rows bound to a workflow model wear its columns (tracker-workflows.js), + // so the header over them must be that model's too (hivecommons/hive#18002: + // hive-default names over cncf-techdocs chevrons). Ungrouped, the + // page is partitioned by model (stable, first-seen order) with a + // header per run; grouped, a group header carries its own columns + // when the sticky header is another model's. The sticky header is + // the lead row's. A hive with no binding resolves one run, no model. + const wfRuns = !grouping && ctx.workflows && typeof stageTrackerWorkflowRuns === 'function' ? stageTrackerWorkflowRuns(page.rows, ctx) : null; + if (wfRuns && wfRuns.length > 1) page.rows = wfRuns.reduce((all, run) => all.concat(run.rows), []); + const leadRepo = page.rows.length ? (grouping ? page.rows[0].group.repo : page.rows[0].repo) : ''; + ctx.leadWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, leadRepo) : null; // A folded secondary is not filtered out: it counts toward the total. if (foldedShared > 0) page.total += foldedShared; // The rows this render draws (this page, group headers aside), for @@ -192,7 +203,9 @@ : itemRow(pageItems[i], i); const trainRows = stageTrackerTrainRows(page.rows, rowOf, typeof stageTrackerTrainWhere === 'function' ? stageTrackerTrainWhere(ordered, d.items) : null); const inlineRows = typeof stageTrackerPreparationInlineRows === 'function' ? stageTrackerPreparationInlineRows(trainRows, pageItems, stages) : trainRows; - const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: stageTrackerHeaderHtml(stages) }].concat(inlineRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); + const runRows = wfRuns && wfRuns.length > 1 ? stageTrackerWorkflowRunHeaderRows(inlineRows, wfRuns, stages) : inlineRows; + const headHtml = typeof stageTrackerColumnsHeaderHtml === 'function' ? stageTrackerColumnsHeaderHtml(ctx.leadWorkflow, stages) : stageTrackerHeaderHtml(stages); + const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: headHtml }].concat(runRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); const rec = stageTrackerReconcile(el, rows, { effect, hidden: document.hidden }); if (rec.deferred) stageTrackerScheduleSettle(); // Selection is keyed on the row, never baked into its HTML, so a kept diff --git a/src/pkg/dashboard/static-src/js/tracker-row-render.js b/src/pkg/dashboard/static-src/js/tracker-row-render.js index 98c6f556f9..5c394ade3f 100644 --- a/src/pkg/dashboard/static-src/js/tracker-row-render.js +++ b/src/pkg/dashboard/static-src/js/tracker-row-render.js @@ -74,7 +74,7 @@ ${stageTrackerRepoChipHtml(g.repo, chipCtx)}<span class="ham-group-name">${esc(g.repo)}</span>${groupWorkflow ? stageTrackerWorkflowBadgeHtml(groupWorkflow) : ''} <span class="ham-group-n">${g.count} ${g.count === 1 ? 'row' : 'rows'}</span>${attn} ${groupWorkflow ? '' : `<span class="ham-group-dist" role="img" aria-label="${esc(g.cols.map((c, i) => stageTrackerColName(i, stages) + ' ' + c).join(', '))}">${dist}</span>`} - </div>${groupWorkflow && !g.collapsed ? stageTrackerWorkflowHeaderHtml(groupWorkflow) : ''}`; + </div>${typeof stageTrackerGroupColumnsHtml === 'function' ? stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) : ''}`; } // Row = title line (number, title, chips, since) + thin strip. Non-active diff --git a/src/pkg/dashboard/static-src/js/tracker-workflows.js b/src/pkg/dashboard/static-src/js/tracker-workflows.js index 03b8e6d24a..fa24eacf3d 100644 --- a/src/pkg/dashboard/static-src/js/tracker-workflows.js +++ b/src/pkg/dashboard/static-src/js/tracker-workflows.js @@ -71,6 +71,62 @@ return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } + // Pure: the key a row's model resolves to ('' for hive-default). + function stageTrackerWorkflowKey(wf) { return wf ? wf.name + '@' + wf.version : ''; } + + // Pure: the column header the rows under it follow — the model's own + // when they are bound to one, else hive-default's. The one path for + // the sticky header and the run headers, so the names above a strip + // are always the names its chevrons wear (hivecommons/hive#18002: + // hive-default's "PR in progress / Merge queue" over model chevrons). + function stageTrackerColumnsHeaderHtml(wf, stages) { + return wf ? stageTrackerWorkflowHeaderHtml(wf) : stageTrackerHeaderHtml(stages); + } + + // Pure: an ungrouped page's rows partitioned, stably and in first-seen + // order, by the model each row's repository follows: [{ wf, key, rows }]. + // One run when every row follows the same model (or none does), so a + // hive-default page is left exactly as sorted. + function stageTrackerWorkflowRuns(rows, ctx) { + const runs = []; + const by = {}; + for (const it of rows || []) { + const wf = stageTrackerWorkflowOf(ctx, it.repo); + const key = stageTrackerWorkflowKey(wf); + if (!by[key]) { by[key] = { wf, key, rows: [] }; runs.push(by[key]); } + by[key].rows.push(it); + } + return runs; + } + + // Run headers are keyed under the group prefix so the selection and + // the reconcile treat them as headers, never as item rows. + const STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX = STAGE_TRACKER_GROUP_KEY_PREFIX + 'workflow:'; + + // Pure: the page's keyed rows with a column header spliced in before + // each run after the first (the first run sits under the sticky + // header). A run whose lead row the train/preparation wrappers did not + // keep by key gets no header rather than a misplaced one. + function stageTrackerWorkflowRunHeaderRows(rows, runs, stages) { + const out = (rows || []).slice(); + (runs || []).slice(1).forEach(run => { + const lead = run.rows[0] ? stageTrackerItemKey(run.rows[0]) : null; + const at = out.findIndex(r => r.key === lead); + if (at >= 0) out.splice(at, 0, { key: STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX + (run.key || 'hive-default'), html: stageTrackerColumnsHeaderHtml(run.wf, stages) }); + }); + return out; + } + + // Under a group header: a bound repository's own column header; a + // hive-default repository's only when the sticky header above is a + // model's (ctx.leadWorkflow), so its rows are not read against that + // model's names. Collapsed groups draw none. + function stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) { + if (!g || g.collapsed) return ''; + if (groupWorkflow) return stageTrackerWorkflowHeaderHtml(groupWorkflow); + return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 8a6db5eb16..22024fe971 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -19932,6 +19932,62 @@ <h3 class="faq-group-title">Where to get help</h3> return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } + // Pure: the key a row's model resolves to ('' for hive-default). + function stageTrackerWorkflowKey(wf) { return wf ? wf.name + '@' + wf.version : ''; } + + // Pure: the column header the rows under it follow — the model's own + // when they are bound to one, else hive-default's. The one path for + // the sticky header and the run headers, so the names above a strip + // are always the names its chevrons wear (hivecommons/hive#18002: + // hive-default's "PR in progress / Merge queue" over model chevrons). + function stageTrackerColumnsHeaderHtml(wf, stages) { + return wf ? stageTrackerWorkflowHeaderHtml(wf) : stageTrackerHeaderHtml(stages); + } + + // Pure: an ungrouped page's rows partitioned, stably and in first-seen + // order, by the model each row's repository follows: [{ wf, key, rows }]. + // One run when every row follows the same model (or none does), so a + // hive-default page is left exactly as sorted. + function stageTrackerWorkflowRuns(rows, ctx) { + const runs = []; + const by = {}; + for (const it of rows || []) { + const wf = stageTrackerWorkflowOf(ctx, it.repo); + const key = stageTrackerWorkflowKey(wf); + if (!by[key]) { by[key] = { wf, key, rows: [] }; runs.push(by[key]); } + by[key].rows.push(it); + } + return runs; + } + + // Run headers are keyed under the group prefix so the selection and + // the reconcile treat them as headers, never as item rows. + const STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX = STAGE_TRACKER_GROUP_KEY_PREFIX + 'workflow:'; + + // Pure: the page's keyed rows with a column header spliced in before + // each run after the first (the first run sits under the sticky + // header). A run whose lead row the train/preparation wrappers did not + // keep by key gets no header rather than a misplaced one. + function stageTrackerWorkflowRunHeaderRows(rows, runs, stages) { + const out = (rows || []).slice(); + (runs || []).slice(1).forEach(run => { + const lead = run.rows[0] ? stageTrackerItemKey(run.rows[0]) : null; + const at = out.findIndex(r => r.key === lead); + if (at >= 0) out.splice(at, 0, { key: STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX + (run.key || 'hive-default'), html: stageTrackerColumnsHeaderHtml(run.wf, stages) }); + }); + return out; + } + + // Under a group header: a bound repository's own column header; a + // hive-default repository's only when the sticky header above is a + // model's (ctx.leadWorkflow), so its rows are not read against that + // model's names. Collapsed groups draw none. + function stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) { + if (!g || g.collapsed) return ''; + if (groupWorkflow) return stageTrackerWorkflowHeaderHtml(groupWorkflow); + return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber @@ -20912,7 +20968,7 @@ <h3 class="faq-group-title">Where to get help</h3> ${stageTrackerRepoChipHtml(g.repo, chipCtx)}<span class="ham-group-name">${esc(g.repo)}</span>${groupWorkflow ? stageTrackerWorkflowBadgeHtml(groupWorkflow) : ''} <span class="ham-group-n">${g.count} ${g.count === 1 ? 'row' : 'rows'}</span>${attn} ${groupWorkflow ? '' : `<span class="ham-group-dist" role="img" aria-label="${esc(g.cols.map((c, i) => stageTrackerColName(i, stages) + ' ' + c).join(', '))}">${dist}</span>`} - </div>${groupWorkflow && !g.collapsed ? stageTrackerWorkflowHeaderHtml(groupWorkflow) : ''}`; + </div>${typeof stageTrackerGroupColumnsHtml === 'function' ? stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) : ''}`; } // Row = title line (number, title, chips, since) + thin strip. Non-active @@ -23545,6 +23601,17 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> ? stageTrackerGroupPage(stageTrackerGroups(ordered, { primary: stageTrackerPrimaryRepo(), collapsed: _stageTrackerCollapsed, stages }), f) : stageTrackerPage(ordered, f); if (!tv && page.page !== f.page) f.page = page.page; + // Rows bound to a workflow model wear its columns (tracker-workflows.js), + // so the header over them must be that model's too (hivecommons/hive#18002: + // hive-default names over cncf-techdocs chevrons). Ungrouped, the + // page is partitioned by model (stable, first-seen order) with a + // header per run; grouped, a group header carries its own columns + // when the sticky header is another model's. The sticky header is + // the lead row's. A hive with no binding resolves one run, no model. + const wfRuns = !grouping && ctx.workflows && typeof stageTrackerWorkflowRuns === 'function' ? stageTrackerWorkflowRuns(page.rows, ctx) : null; + if (wfRuns && wfRuns.length > 1) page.rows = wfRuns.reduce((all, run) => all.concat(run.rows), []); + const leadRepo = page.rows.length ? (grouping ? page.rows[0].group.repo : page.rows[0].repo) : ''; + ctx.leadWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, leadRepo) : null; // A folded secondary is not filtered out: it counts toward the total. if (foldedShared > 0) page.total += foldedShared; // The rows this render draws (this page, group headers aside), for @@ -23620,7 +23687,9 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> : itemRow(pageItems[i], i); const trainRows = stageTrackerTrainRows(page.rows, rowOf, typeof stageTrackerTrainWhere === 'function' ? stageTrackerTrainWhere(ordered, d.items) : null); const inlineRows = typeof stageTrackerPreparationInlineRows === 'function' ? stageTrackerPreparationInlineRows(trainRows, pageItems, stages) : trainRows; - const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: stageTrackerHeaderHtml(stages) }].concat(inlineRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); + const runRows = wfRuns && wfRuns.length > 1 ? stageTrackerWorkflowRunHeaderRows(inlineRows, wfRuns, stages) : inlineRows; + const headHtml = typeof stageTrackerColumnsHeaderHtml === 'function' ? stageTrackerColumnsHeaderHtml(ctx.leadWorkflow, stages) : stageTrackerHeaderHtml(stages); + const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: headHtml }].concat(runRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); const rec = stageTrackerReconcile(el, rows, { effect, hidden: document.hidden }); if (rec.deferred) stageTrackerScheduleSettle(); // Selection is keyed on the row, never baked into its HTML, so a kept From 1ecfd39eb1202c0c523a810def7af2893a7a914b Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 16:06:40 -0700 Subject: [PATCH 125/223] =?UTF-8?q?=F0=9F=90=9B=20spoke:=20install=20missi?= =?UTF-8?q?ng=20startupProbe=20on=20self-upgrade=20so=20forge-blocked=20bo?= =?UTF-8?q?ots=20are=20not=20liveness-killed=20(#18010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 spoke: install missing startupProbe on self-upgrade so forge-blocked boots are not liveness-killed Pre-#9692 hosted Deployments on pull-only clusters (hub has no kubectl reach) still have no startupProbe. With github.ibm.com unreachable the spoke's boot (App token mint, advisory ensure, coverage classify) takes longer than the 3x30s livenessProbe, so kubelet SIGTERMs it before the dashboard binds and the pod crash-loops for the whole outage — the hub then shows the row offline (grey dot). The spoke-side backfill explicitly skipped containers with no probe; it now installs the template's full probe (port taken from the existing readiness/liveness probe, else 3002). Also: the fleet status-dot hovers gain a colour legend and the grey dot says 'no heartbeat received in the last 5 min'. Signed-off-by: Andy Anderson <andy@clubanderson.com> * test: pin new offline-dot title and give the no-restamp fixture the template probe Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ed-spoke-installs-missing-startup-probe.md | 1 + src/pkg/hub/assets/dashboard.html | 11 +++- src/pkg/hub/offline_dot_tooltip_test.go | 3 +- src/pkg/hub/spoke/self_startup_probe.go | 64 +++++++++++++++++-- src/pkg/hub/spoke/self_startup_probe_test.go | 20 +++++- .../hub/spoke/self_upgrade_coverage_test.go | 6 +- 6 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 changelog.d/fixed-spoke-installs-missing-startup-probe.md diff --git a/changelog.d/fixed-spoke-installs-missing-startup-probe.md b/changelog.d/fixed-spoke-installs-missing-startup-probe.md new file mode 100644 index 0000000000..3680f325a5 --- /dev/null +++ b/changelog.d/fixed-spoke-installs-missing-startup-probe.md @@ -0,0 +1 @@ +- Spoke self-upgrade now installs the hosted template's startupProbe (10-min budget) on a hive Deployment that has none, not only raises an undersized one — pre-#9692 spokes on pull-only clusters were liveness-killed 90s into a forge-blocked boot and crash-looped (grey dot) for the length of the GHE outage; the fleet status-dot hovers now carry a colour legend and the grey dot says "no heartbeat in 5 min". diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 220d0efbad..2e910c1e5e 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -4127,6 +4127,9 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) rows + viewAll; } + /* One-line colour key appended to the status-dot hovers so the dot is + self-explaining. Mirrors the healthBadge() palette + the offline grey. */ + var STATUS_DOT_LEGEND = 'Dot colours: green ok \u00b7 amber warning/stale \u00b7 red degraded/critical \u00b7 blue upgrading \u00b7 grey offline (no heartbeat >5 min)'; function healthBadge(h) { var hp = h.health || {}; var st = hp.status || 'unknown'; @@ -4227,6 +4230,7 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) if (hns) { lines.push('ns: ' + hns); } + lines.push(STATUS_DOT_LEGEND); var access = h.access || []; var dotMarkup = isUpgrading && !isDeleting ? '<span class="online-dot upgrading" style="margin-right:var(--sp-0)"></span>' @@ -8919,13 +8923,18 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) // cluster. Built defensively \u2014 every part is skipped when its field is // empty \u2014 and escaped with escAttr since it lands inside a quoted title. var offlineDotTitle = (function() { - var parts = ['Offline']; + /* Say WHAT grey means, not just "Offline": the hub flips a row + offline when no heartbeat arrives within maxHeartbeatAge (5 min, + pkg/hub/server.go), which operators kept reading as "unknown + health" rather than "the spoke stopped talking". */ + var parts = ['Offline \u2014 no heartbeat received in the last 5 min (grey dot)']; var ns = hiveNamespace(h); // 'hive-hosted-<id>' or '' for local if (ns) parts.push('ns: ' + ns); var cluster = h.clusterName || h.clusterId || ''; if (cluster) parts.push('cluster: ' + cluster); if (h.lastHeartbeat) parts.push('last seen: ' + fmtUserTS(h.lastHeartbeat)); else if (h.offlineSince) parts.push('offline since: ' + fmtUserTS(h.offlineSince)); + parts.push(STATUS_DOT_LEGEND); return parts.join('\n'); })(); var dot = h.upgrading diff --git a/src/pkg/hub/offline_dot_tooltip_test.go b/src/pkg/hub/offline_dot_tooltip_test.go index f0c0ae68f3..786e97d31b 100644 --- a/src/pkg/hub/offline_dot_tooltip_test.go +++ b/src/pkg/hub/offline_dot_tooltip_test.go @@ -30,7 +30,8 @@ func TestOfflineStatusDotCarriesTooltip(t *testing.T) { // cluster and last-seen lines, each skipped when its field is empty. for _, part := range []string{ `var offlineDotTitle = (function() {`, - `var parts = ['Offline'];`, + `var parts = ['Offline \u2014 no heartbeat received in the last 5 min (grey dot)'];`, + `parts.push(STATUS_DOT_LEGEND);`, `var ns = hiveNamespace(h);`, `if (ns) parts.push('ns: ' + ns);`, `var cluster = h.clusterName || h.clusterId || '';`, diff --git a/src/pkg/hub/spoke/self_startup_probe.go b/src/pkg/hub/spoke/self_startup_probe.go index 980d7cf95c..88d9cca758 100644 --- a/src/pkg/hub/spoke/self_startup_probe.go +++ b/src/pkg/hub/spoke/self_startup_probe.go @@ -13,12 +13,24 @@ const ( HostedStartupProbeInitialDelaySeconds = 10 HostedStartupProbePeriodSeconds = 5 HostedStartupProbeFailureThreshold = (int(HostedStartupBudget/time.Second) - HostedStartupProbeInitialDelaySeconds) / HostedStartupProbePeriodSeconds + + // HostedStartupProbePath and HostedDashboardPort are the hosted template's + // startupProbe handler, used when a hive container has NO startupProbe at + // all and the whole probe has to be installed (not just its threshold). + HostedStartupProbePath = "/api/health" + HostedDashboardPort = 3002 ) // k8sUpgradePatch backfills the startup budget in the SAME template update as -// the image/restart annotation, avoiding a second rollout. Only raise an existing -// hive probe: sidecars, init containers and deployments without a probe keep -// their configuration. Strategic merge preserves the handler and other timings. +// the image/restart annotation, avoiding a second rollout. Only the hive +// container is touched: sidecars and init containers keep their configuration. +// An existing hive probe is raised to the template budget (strategic merge +// preserves its handler and other timings). A hive container with NO +// startupProbe gets the template's full probe: pre-#9692 deployments on +// pull-only clusters (where the hub cannot reach the Deployment itself) were +// left with only the 3×30s livenessProbe, so a boot stalled >90s behind forge +// timeouts was SIGTERMed before the dashboard bound and the spoke crash-looped +// for the length of the forge outage. // A nil body requests only a probe backfill and skips PATCH when none is needed. func k8sUpgradePatch(path string, body []byte) error { current, err := k8sAPIGet(path) @@ -41,6 +53,8 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { StartupProbe *struct { FailureThreshold int `json:"failureThreshold"` } `json:"startupProbe"` + ReadinessProbe *probeHandler `json:"readinessProbe"` + LivenessProbe *probeHandler `json:"livenessProbe"` } `json:"containers"` } `json:"spec"` } `json:"template"` @@ -51,7 +65,7 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { } probeOnly := body == nil for _, c := range dep.Spec.Template.Spec.Containers { - if c.Name != selfUpgradeDeployName || c.StartupProbe == nil || c.StartupProbe.FailureThreshold >= HostedStartupProbeFailureThreshold { + if c.Name != selfUpgradeDeployName || (c.StartupProbe != nil && c.StartupProbe.FailureThreshold >= HostedStartupProbeFailureThreshold) { continue } if probeOnly { @@ -82,7 +96,11 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { target = map[string]any{"name": c.Name} containers = append(containers, target) } - target["startupProbe"] = map[string]any{"failureThreshold": HostedStartupProbeFailureThreshold} + if c.StartupProbe == nil { + target["startupProbe"] = fullStartupProbe(startupProbePort(c.ReadinessProbe, c.LivenessProbe)) + } else { + target["startupProbe"] = map[string]any{"failureThreshold": HostedStartupProbeFailureThreshold} + } spec["containers"] = containers body, err = json.Marshal(patch) if err != nil { @@ -95,3 +113,39 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { } return k8sAPIPatch(path, body) } + +// probeHandler is the slice of a Kubernetes probe the backfill needs: the +// httpGet port the dashboard already answers on. +type probeHandler struct { + HTTPGet *struct { + Port json.Number `json:"port"` + } `json:"httpGet"` +} + +// startupProbePort picks the port for a newly installed startupProbe: the +// port the existing readiness or liveness probe already targets (the dashboard +// is proven to answer there), else the hosted template default. +func startupProbePort(probes ...*probeHandler) any { + for _, p := range probes { + if p == nil || p.HTTPGet == nil || p.HTTPGet.Port == "" { + continue + } + if n, err := p.HTTPGet.Port.Int64(); err == nil && n > 0 { + return n + } + return p.HTTPGet.Port.String() + } + return HostedDashboardPort +} + +// fullStartupProbe renders the hosted template's startupProbe for a container +// that has none, so the self-upgrade path converges exactly what provisioning +// would have written. +func fullStartupProbe(port any) map[string]any { + return map[string]any{ + "httpGet": map[string]any{"path": HostedStartupProbePath, "port": port}, + "initialDelaySeconds": HostedStartupProbeInitialDelaySeconds, + "periodSeconds": HostedStartupProbePeriodSeconds, + "failureThreshold": HostedStartupProbeFailureThreshold, + } +} diff --git a/src/pkg/hub/spoke/self_startup_probe_test.go b/src/pkg/hub/spoke/self_startup_probe_test.go index 1b9a11d052..c189db572a 100644 --- a/src/pkg/hub/spoke/self_startup_probe_test.go +++ b/src/pkg/hub/spoke/self_startup_probe_test.go @@ -20,7 +20,10 @@ func TestSelfUpgradeBackfillsStartupProbe(t *testing.T) { {"stale", `,"startupProbe":{"httpGet":{"path":"/api/health","port":3002},"failureThreshold":30,"periodSeconds":5}`, true}, {"current", `,"startupProbe":{"failureThreshold":118}`, false}, {"larger", `,"startupProbe":{"failureThreshold":200}`, false}, - {"absent", ``, false}, + // No probe at all: install the template's full probe. The port + // follows the existing readiness probe when one is present. + {"absent", ``, true}, + {"absent-with-readiness", `,"readinessProbe":{"httpGet":{"path":"/api/health","port":4100}}`, true}, } { t.Run(operation+"/"+probe.name, func(t *testing.T) { probeJSON := probe.json @@ -105,9 +108,22 @@ func TestSelfUpgradeBackfillsStartupProbe(t *testing.T) { for _, c := range patch.Spec.Template.Spec.Containers { if c.Name == "hive" { found = c.Probe != nil - if found && (len(c.Probe) != 1 || c.Probe["failureThreshold"] != float64(118)) { + if found && c.Probe["failureThreshold"] != float64(118) { t.Errorf("unexpected probe patch: %v", c.Probe) } + if found && strings.HasPrefix(probe.name, "absent") { + wantPort := float64(3002) + if probe.name == "absent-with-readiness" { + wantPort = 4100 + } + hg, _ := c.Probe["httpGet"].(map[string]any) + if hg == nil || hg["path"] != "/api/health" || hg["port"] != wantPort || + c.Probe["periodSeconds"] != float64(5) || c.Probe["initialDelaySeconds"] != float64(10) { + t.Errorf("missing probe must be installed in full (port %v): %v", wantPort, c.Probe) + } + } else if found && len(c.Probe) != 1 { + t.Errorf("existing probe must only have its threshold raised: %v", c.Probe) + } if operation == "image" && c.Image != "ghcr.io/hivecommons/hive:candidate" { t.Error("lost image update") } diff --git a/src/pkg/hub/spoke/self_upgrade_coverage_test.go b/src/pkg/hub/spoke/self_upgrade_coverage_test.go index f64e521120..1ca7beb9e3 100644 --- a/src/pkg/hub/spoke/self_upgrade_coverage_test.go +++ b/src/pkg/hub/spoke/self_upgrade_coverage_test.go @@ -269,12 +269,14 @@ func TestSwitchImageSelfSuccess(t *testing.T) { // template each time — so the mid-init pod was killed and replaced on every // beat (observed live: 18 ReplicaSets in 8 minutes, none reaching Ready). A // Deployment whose containers all already carry the target must not be -// PATCHed again. +// PATCHed again. The fixture carries the template startupProbe: a hive +// container with NO probe is the one case that still warrants a (one-time, +// annotation-free) PATCH mid-flight — see TestSelfUpgradeBackfillsStartupProbe. func TestSwitchImageSelfDoesNotRestampWhenAlreadyAtImage(t *testing.T) { const target = "ghcr.io/hivecommons/hive:candidate" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet { - w.Write([]byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive","image":"` + target + `"}],"initContainers":[{"name":"init","image":"` + target + `"}]}}}}`)) + w.Write([]byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive","image":"` + target + `","startupProbe":{"failureThreshold":118}}],"initContainers":[{"name":"init","image":"` + target + `"}]}}}}`)) return } t.Errorf("deployment already at %s, but a %s was issued — that re-rolls the pod mid-init", target, r.Method) From 19afdf06d56b55961bbf103f7cc7f6e30607aa76 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 23:19:37 +0000 Subject: [PATCH 126/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.238.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.238.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 8 ++++++++ changelog.d/fixed-claim-forge-switch-discovery.md | 1 - changelog.d/fixed-claim-pending-forge-pill.md | 1 - changelog.d/fixed-sse-drop-warn-flood.md | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/fixed-claim-forge-switch-discovery.md delete mode 100644 changelog.d/fixed-claim-pending-forge-pill.md delete mode 100644 changelog.d/fixed-sse-drop-warn-flood.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 41885cd023..bf271ee566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.238.1) + +### Fixed + +- Spoke: a placeholder claimed onto a different forge than its pool default (GHE slot → github.com repo) now moves GitHub App installation discovery to the delivered App and API URL even while installation_id is still 0, instead of polling the boot-time App/forge forever and reporting "App 5686 has no installation" until a pod restart. +- Hub fleet view: a placeholder assigned with an explicit forge now shows that forge (e.g. github.com) in the Location pill while the claim is pending, instead of the cluster default the spoke still heartbeats from its pre-claim config (github.ibm.com on vllm-d). +- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). + ## 2026-11-12 (v5.238.0) ### Added diff --git a/changelog.d/fixed-claim-forge-switch-discovery.md b/changelog.d/fixed-claim-forge-switch-discovery.md deleted file mode 100644 index 788c262af6..0000000000 --- a/changelog.d/fixed-claim-forge-switch-discovery.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke: a placeholder claimed onto a different forge than its pool default (GHE slot → github.com repo) now moves GitHub App installation discovery to the delivered App and API URL even while installation_id is still 0, instead of polling the boot-time App/forge forever and reporting "App 5686 has no installation" until a pod restart. diff --git a/changelog.d/fixed-claim-pending-forge-pill.md b/changelog.d/fixed-claim-pending-forge-pill.md deleted file mode 100644 index c5b31e94d6..0000000000 --- a/changelog.d/fixed-claim-pending-forge-pill.md +++ /dev/null @@ -1 +0,0 @@ -- Hub fleet view: a placeholder assigned with an explicit forge now shows that forge (e.g. github.com) in the Location pill while the claim is pending, instead of the cluster default the spoke still heartbeats from its pre-claim config (github.ibm.com on vllm-d). diff --git a/changelog.d/fixed-sse-drop-warn-flood.md b/changelog.d/fixed-sse-drop-warn-flood.md deleted file mode 100644 index a5494ff99b..0000000000 --- a/changelog.d/fixed-sse-drop-warn-flood.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). From ba9afc614e79a221adb1195f828f0af7e20cf026 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 16:33:43 -0700 Subject: [PATCH 127/223] =?UTF-8?q?=E2=9C=A8=20project.reference=5Frepos:?= =?UTF-8?q?=20study-only=20repos=20with=20optional=20issues/prs=20opt-in?= =?UTF-8?q?=20(#18015)=20(#18023)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ project.reference_repos: study-only repos with optional issues/prs opt-in (#18015) A hive whose deliverable is about another project (CNCF TechDocs assessing hivecommons/* into cncf/techdocs) can list owner/name repos it may clone unauthenticated and GET. The proxy strips the hive token on those repos and refuses every write by default with a message naming reference_repos; a rare per-entry write: [issues, prs] opt-in allows only open-issue, open-PR/topic branch push and comments on own items, sent with a credential valid for that repo (App installed on the target org, or write_token_file). Boot refuses write-mode without either (exit 22). The kick lists reference repos so the scanner stops flagging such items out-of-scope/critical; dashboard shows a minimal reference · <tier> card. Fixes #18015 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🌱 reference repos: settings-parity allowlist, no new inline styles Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-reference-repos.md | 1 + src/cmd/hive/main.go | 4 + src/cmd/hive/reference_repos.go | 87 ++++ src/cmd/hive/reference_repos_test.go | 46 ++ src/crux/proxy/github_proxy.go | 10 + src/crux/proxy/github_proxy_mitm.go | 28 ++ src/crux/proxy/github_proxy_policy.go | 36 ++ src/crux/proxy/reference_repos.go | 406 ++++++++++++++++++ src/crux/proxy/reference_repos_test.go | 237 ++++++++++ src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/github-write-surface.md | 15 + src/docs/operator-reference/README.md | 3 + src/pkg/config/config.go | 7 + src/pkg/config/reference_repos.go | 231 ++++++++++ src/pkg/config/reference_repos_test.go | 101 +++++ src/pkg/config/validate.go | 3 + src/pkg/dashboard/server.go | 21 +- .../dashboard/settings_parity_allowlist.txt | 3 + src/pkg/dashboard/static-src/css/repos.css | 7 + .../dashboard/static-src/js/repos-render.js | 27 +- src/pkg/dashboard/static/index.html | 34 +- src/pkg/dashboard/status_builder.go | 14 + src/pkg/github/reference_repo_token.go | 190 ++++++++ src/pkg/github/reference_repo_token_test.go | 120 ++++++ src/pkg/scheduler/agent_repo_scope_test.go | 27 ++ src/pkg/scheduler/kickmessage.go | 12 + 29 files changed, 1679 insertions(+), 23 deletions(-) create mode 100644 changelog.d/added-reference-repos.md create mode 100644 src/cmd/hive/reference_repos.go create mode 100644 src/cmd/hive/reference_repos_test.go create mode 100644 src/crux/proxy/reference_repos.go create mode 100644 src/crux/proxy/reference_repos_test.go create mode 100644 src/pkg/config/reference_repos.go create mode 100644 src/pkg/config/reference_repos_test.go create mode 100644 src/pkg/github/reference_repo_token.go create mode 100644 src/pkg/github/reference_repo_token_test.go diff --git a/changelog.d/added-reference-repos.md b/changelog.d/added-reference-repos.md new file mode 100644 index 0000000000..0105e3a710 --- /dev/null +++ b/changelog.d/added-reference-repos.md @@ -0,0 +1 @@ +- **`project.reference_repos`: study-only repos outside the project** ([#18015](https://github.com/hivecommons/hive/issues/18015)). A hive whose deliverable is *about* another project (a docs assessment of `hivecommons/*` landing in `cncf/techdocs`) can list `owner/name` repos it may clone unauthenticated and GET, with the proxy stripping the hive's token and refusing every write by default; the kick tells agents such items are in scope and that findings go in the deliverable in the authorized repo, so the scanner no longer flags them out-of-scope/critical. A rare per-entry `write: [issues, prs]` opt-in allows only opening issues, opening PRs/pushing topic branches and commenting on the hive's own items, using the App's installation on the target org or `write_token_file`; boot refuses write-mode without either (exit 22). Dashboard shows a minimal `reference · <tier>` card. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index ea1821f5e2..e58fe9c42a 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4143,6 +4143,10 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { // unconditionally: with HIVE_PROXY_INJECT_GH_AUTH unset (the default) // the proxy never consults the source and the registry stays empty. b.githubProxy.SetAgentTokenSource(github.AgentProxyToken) + // project.reference_repos (#18015): study-only repos outside the + // project, with a rare per-entry issues/prs opt-in served by a + // credential verified here (boot refuses otherwise). + b.wireReferenceRepos() // Durable attempted-vs-blocked write counters (#12069); observation only. if err := proxy.MigrateWriteCounters(proxy.LegacyWriteCountersPath, proxy.WriteCountersPath, time.Now()); err != nil { b.logger.Warn("write counters: legacy file not migrated", "error", err) diff --git a/src/cmd/hive/reference_repos.go b/src/cmd/hive/reference_repos.go new file mode 100644 index 0000000000..f87c367245 --- /dev/null +++ b/src/cmd/hive/reference_repos.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// referenceRepoCredentialExitCode marks an exit caused by a +// project.reference_repos entry that opted in to writes without a credential +// GitHub would honour for that repo (hivecommons/hive#18015). Its own code so +// the termination state says "install the App on the target org or supply +// write_token_file", not "bug". +const referenceRepoCredentialExitCode = 22 + +// referenceRepoDeps are the seams buildReferenceRepoPolicies needs: App +// installation discovery on a foreign org and token-identity resolution. +// Either may be nil when the hive has no GitHub App auth, in which case a +// write-mode entry can only be served by write_token_file. +type referenceRepoDeps struct { + botLogin string + discover func(ctx context.Context, org string) (int64, error) + whoAmI func(ctx context.Context, token string) (string, error) + auth func() *github.AppAuth +} + +// buildReferenceRepoPolicies turns the config list into proxy policies. A +// read-only entry needs nothing; a write-mode entry is verified now so the +// failure is a boot-time config error naming the repo. +func buildReferenceRepoPolicies(ctx context.Context, refs []config.ReferenceRepo, deps referenceRepoDeps, logger *slog.Logger) ([]proxy.ReferenceRepoPolicy, error) { + out := make([]proxy.ReferenceRepoPolicy, 0, len(refs)) + for _, rr := range refs { + pol := proxy.ReferenceRepoPolicy{Repo: rr.Repo, Issues: rr.WritesIssues(), PRs: rr.WritesPRs()} + if !rr.ReadOnly() { + login, err := github.VerifyReferenceRepoWriteCredential(ctx, rr, deps.botLogin, deps.discover, deps.whoAmI) + if err != nil { + return nil, err + } + pol.Login = login + pol.Token = github.ReferenceRepoTokenSource(rr, deps.auth, logger) + } + out = append(out, pol) + } + return out, nil +} + +// wireReferenceRepos installs project.reference_repos on the proxy, refusing +// to boot when a write-mode entry has no usable credential. +func (b *boot) wireReferenceRepos() { + refs := b.cfg.Project.ReferenceRepos + if len(refs) == 0 || b.githubProxy == nil { + return + } + deps := referenceRepoDeps{auth: func() *github.AppAuth { return nil }} + if b.ghClient != nil { + if auth := b.ghClient.AppAuth(); auth != nil { + apiURL := auth.APIURL() + deps.botLogin = auth.BotLogin() + deps.discover = auth.DiscoverInstallationID + deps.whoAmI = func(ctx context.Context, token string) (string, error) { + return github.TokenUserLogin(ctx, token, apiURL) + } + // Re-fetched per call: the AppAuth is replaced on key rotation. + deps.auth = func() *github.AppAuth { return b.ghClient.AppAuth() } + } else { + deps.whoAmI = func(ctx context.Context, token string) (string, error) { + return github.TokenUserLogin(ctx, token, "") + } + } + } + policies, err := buildReferenceRepoPolicies(b.ctx, refs, deps, b.logger) + if err != nil { + b.logger.Error("refusing to start: project.reference_repos write-mode entry has no usable credential (#18015)", "error", err.Error()) + if errors.Is(err, github.ErrReferenceRepoNoCredential) { + fmt.Fprintln(os.Stderr, err.Error()) + } + os.Exit(referenceRepoCredentialExitCode) + } + b.githubProxy.SetReferenceRepos(policies) + b.logger.Info("reference repos wired", "count", len(policies), "repos", b.cfg.ReferenceRepoSlugs()) +} diff --git a/src/cmd/hive/reference_repos_test.go b/src/cmd/hive/reference_repos_test.go new file mode 100644 index 0000000000..b10d70646a --- /dev/null +++ b/src/cmd/hive/reference_repos_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +func TestBuildReferenceRepoPolicies(t *testing.T) { + ctx := context.Background() + deps := referenceRepoDeps{ + botLogin: "hive[bot]", + discover: func(_ context.Context, org string) (int64, error) { + if org == "hivecommons" { + return 1, nil + } + return 0, github.ErrNoInstallationForOrg + }, + auth: func() *github.AppAuth { return nil }, + } + refs := []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + pols, err := buildReferenceRepoPolicies(ctx, refs, deps, testLogger()) + if err != nil { + t.Fatal(err) + } + if len(pols) != 2 { + t.Fatalf("got %d policies", len(pols)) + } + if pols[0].Token != nil || pols[0].Issues || pols[0].PRs || pols[0].Login != "" { + t.Errorf("read-only policy carries write state: %+v", pols[0]) + } + if pols[1].Token == nil || !pols[1].Issues || !pols[1].PRs || pols[1].Login != "hive[bot]" { + t.Errorf("write policy incomplete: %+v", pols[1]) + } + + _, err = buildReferenceRepoPolicies(ctx, []config.ReferenceRepo{{Repo: "kubernetes/website", Write: []string{"issues"}}}, deps, testLogger()) + if !errors.Is(err, github.ErrReferenceRepoNoCredential) { + t.Fatalf("uninstalled org must refuse with ErrReferenceRepoNoCredential, got %v", err) + } +} diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index c47abfedf8..5b616e62b2 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -86,6 +86,16 @@ type GitHubProxy struct { logger *slog.Logger uidMap *agent.UIDMap allowedRepos map[string]bool + // Reference repos (#18015): study-only repositories outside allowedRepos, + // read-only by default with a per-entry issues/prs opt-in. Reads to them + // are forwarded without the hive's credential; writes are gated by + // referenceRepoRefusal. referenceDefaults caches each repo's default + // branch for the push guard; referenceLookup is the GET those checks use + // (nil = defaultReferenceLookup; tests inject one). + referenceMu sync.RWMutex + referenceRepos map[string]ReferenceRepoPolicy + referenceDefaults map[string]referenceDefaultBranch + referenceLookup referenceLookupFunc // repoPaused reports whether a repo ("owner/repo", as ExtractRepo yields it) // is under an operator pause (#6203). Guarded because the dashboard can diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go index da81467ffe..d750bd976e 100644 --- a/src/crux/proxy/github_proxy_mitm.go +++ b/src/crux/proxy/github_proxy_mitm.go @@ -77,7 +77,29 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str // refused and audited as agent_write_refused. Checked after pause and // scope so their more specific reasons win when both apply. writeSurfaceKind := "" + // Reference repos (#18015) sit outside every list below: not in + // allowedRepos (so the mode chain's repo filter would refuse the + // write with the wrong reason), not pausable, not agent-scoped. The + // gate runs first so its refusal — naming reference_repos and the + // tier — is what the agent reads, and an allowed opt-in write is + // remembered so the repo filter and the PR hard-deny below do not + // undo it (the credential it will carry is the reference repo's + // own, so App-bot authorship is already guaranteed). + referenceWrite := false if agentName != internalCallerName { + if pol, isRef := p.referenceRepoFor(repoScopePath(req)); isRef { + if reason, refused := p.referenceRepoRefusal(req); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonReferenceRepo + } else { + referenceWrite = referenceWriteCredential(pol, req.Method, req.URL.Path, req.URL.RawQuery) + } + } + } + if blocked { + // Refused as a reference-repo write; nothing below can un-block it. + } else if agentName != internalCallerName { if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { blocked = true blockReason = reason @@ -165,6 +187,12 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str // autonomy. Its control-plane calls (App token mint, heartbeat) // must not be gated as if they were agent writes. Repo-filter and // canary-egress checks below still apply. + } else if referenceWrite && !referenceWriteAllowedByMode(mode, req.Method, req.URL.Path) { + blocked = true + blockCode = DenyReasonMode + } else if referenceWrite { + // Opt-in reference write at a sufficient mode: skip the PR + // hard-deny and the repo filter, both of which assume a managed repo. } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { blocked = true blockCode = DenyReasonMode diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go index b61b8d03ab..2d60d115e4 100644 --- a/src/crux/proxy/github_proxy_policy.go +++ b/src/crux/proxy/github_proxy_policy.go @@ -180,6 +180,17 @@ const loginPathPrefix = "/login/" func (p *GitHubProxy) rewriteGitHubAuth(req *http.Request, agentName string) { // NOTE: callers must gate on injectsGitHubAuthForHost first; this function // does not know which host the request is bound for. + if agentName != internalCallerName { + if pol, isRef := p.referenceRepoFor(repoScopePath(req)); isRef { + // Reference repo (#18015): the agent's own credential belongs to + // the hive's installation and is never valid here, so it is + // stripped whatever the injection flag says. A read goes out + // unauthenticated, as the public internet would send it; an + // opt-in write carries the repo's own write credential. + p.rewriteReferenceRepoAuth(req, agentName, pol) + return + } + } if !p.injectGHAuth { // Flag off (the default): behavior byte-identical to before #1861 — // no header is touched on any path. @@ -372,3 +383,28 @@ func isGitPath(path string) bool { strings.HasSuffix(path, "/git-upload-pack") || strings.HasSuffix(path, "/info/refs") } + +func (p *GitHubProxy) rewriteReferenceRepoAuth(req *http.Request, agentName string, pol ReferenceRepoPolicy) { + req.Header.Del("Authorization") + if isGitPath(req.URL.Path) { + // Same reason as the managed-repo git branch above: each git request + // must re-enter this rewrite on its own connection. + req.Close = true + } + if !referenceWriteCredential(pol, req.Method, req.URL.Path, req.URL.RawQuery) { + p.logger.Debug("proxy auth: reference repo read forwarded without credentials", "agent", agentName, "repo", pol.Repo, "injected", false) + return + } + token := p.referenceToken(pol) + if token == "" { + p.logger.Warn("proxy auth: reference repo write has no credential — forwarding without one (check the write_token_file or the App installation on the target org)", "agent", agentName, "repo", pol.Repo, "injected", false) + return + } + if isGitPath(req.URL.Path) { + basic := base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser + ":" + token)) + req.Header.Set("Authorization", "Basic "+basic) + } else { + req.Header.Set("Authorization", "token "+token) + } + p.logger.Debug("proxy auth: reference repo write credential attached", "agent", agentName, "repo", pol.Repo, "injected", true) +} diff --git a/src/crux/proxy/reference_repos.go b/src/crux/proxy/reference_repos.go new file mode 100644 index 0000000000..be69c0c531 --- /dev/null +++ b/src/crux/proxy/reference_repos.go @@ -0,0 +1,406 @@ +package proxy + +import ( + "bufio" + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/agent" +) + +// Reference repositories (hivecommons/hive#18015). +// +// A reference repo is one the hive's agents may STUDY without managing it: +// project.reference_repos. Reads — git clone/fetch and GitHub GETs — pass, and +// pass WITHOUT the hive's credential: the agent's injected token belongs to the +// hive's own installation and is not valid for a foreign repo, so it is +// stripped and the request goes out as the public internet would send it. +// +// Writes are refused unless the entry opted in, and even then only to the +// narrow surface an assessor needs: open an issue, open a PR from a topic +// branch, comment on the hive's OWN issue or PR. Merging, pushing the default +// branch, labels/assignees/closing on anyone's items and reviews are never +// allowed — there is no mode, capability or relay that widens this. The write +// credential is the policy's own Token source (an installation token minted +// on the TARGET org, or an operator-supplied file), never the hive's token. + +// DenyReasonReferenceRepo is the 403 reason code for a refused reference-repo +// write (see denial_message.go for the vocabulary). +const DenyReasonReferenceRepo = "reference_repo" + +// ReferenceRepoPolicy is the proxy's view of one project.reference_repos entry. +type ReferenceRepoPolicy struct { + // Repo is the "owner/name" slug. + Repo string + // Issues and PRs are the write opt-ins. + Issues, PRs bool + // Token yields the write credential for Repo. nil for a read-only entry. + Token func() (string, bool) + // Login is the GitHub login the Token acts as ("<app>[bot]" or the PAT's + // user); own-item checks compare an item's author against it. + Login string +} + +func (r ReferenceRepoPolicy) readOnly() bool { return !r.Issues && !r.PRs } + +func (r ReferenceRepoPolicy) tier() string { + switch { + case r.Issues && r.PRs: + return "issues+prs" + case r.Issues: + return "issues" + case r.PRs: + return "prs" + } + return "read-only" +} + +// referenceDecision is the outcome of the path-only part of the write check. +type referenceDecision int + +const ( + referenceNotReference referenceDecision = iota // not a reference repo, or a read: nothing to do + referenceAllow + referenceDeny + referenceOwnItem // allowed only if item #N was authored by the policy's Login + referencePush // git push: allowed only when no ref targets the default branch +) + +var ( + refIssuesPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) + refPullsPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`) + refItemCommentsPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/(?:issues|pulls)/(\d+)/comments$`) + refGitReceivePackRe = regexp.MustCompile(`\.git/git-receive-pack$`) + refGitInfoRefsRe = regexp.MustCompile(`\.git/info/refs$`) + referenceLookupLimit = int64(1 << 20) +) + +// ReferenceRepoDecision classifies one request against a reference-repo +// policy without touching the network. number is the item for an +// referenceOwnItem decision. reason is the agent-facing refusal for +// referenceDeny. +func ReferenceRepoDecision(pol ReferenceRepoPolicy, method, path, rawQuery string) (d referenceDecision, number int, reason string) { + push := refGitReceivePackRe.MatchString(path) || (refGitInfoRefsRe.MatchString(path) && gitServiceIsReceivePack(rawQuery)) + if !push && (!writeMethods[method] || gitUploadPackPath.MatchString(path)) { + return referenceNotReference, 0, "" + } + if pol.readOnly() { + return referenceDeny, 0, referenceRepoRefusal(pol, "it is listed as READ-ONLY") + } + switch { + case push: + if !pol.PRs { + return referenceDeny, 0, referenceRepoRefusal(pol, "pushing needs write: [prs]") + } + return referencePush, 0, "" + case method == "POST" && refIssuesPath.MatchString(path): + if !pol.Issues { + return referenceDeny, 0, referenceRepoRefusal(pol, "opening an issue needs write: [issues]") + } + return referenceAllow, 0, "" + case method == "POST" && refPullsPath.MatchString(path): + if !pol.PRs { + return referenceDeny, 0, referenceRepoRefusal(pol, "opening a PR needs write: [prs]") + } + return referenceAllow, 0, "" + case method == "POST" && refItemCommentsPath.MatchString(path): + m := refItemCommentsPath.FindStringSubmatch(path) + n, _ := strconv.Atoi(m[1]) + return referenceOwnItem, n, "" + } + return referenceDeny, 0, referenceRepoRefusal(pol, "only opening an issue or PR and commenting on the hive's own items are ever allowed there — never merge, push to the default branch, label, assign, close or review") +} + +func referenceRepoRefusal(pol ReferenceRepoPolicy, why string) string { + return "repository " + pol.Repo + " is a REFERENCE repo (project.reference_repos, tier " + pol.tier() + "): " + why + ". Reads — clone, fetch, GET — still work. Put findings about it into the deliverable in your AUTHORIZED repo instead. Do NOT retry, do NOT route around this with another CLI, the hive relays or the GitHub MCP, and do NOT file an issue about it: this is how the operator scoped the hive, not an outage." +} + +// SetReferenceRepos installs the reference-repo policies (#18015). Replaces +// the previous set; nil clears it. +func (p *GitHubProxy) SetReferenceRepos(policies []ReferenceRepoPolicy) { + m := make(map[string]ReferenceRepoPolicy, len(policies)) + for _, pol := range policies { + key := strings.ToLower(strings.TrimSpace(pol.Repo)) + if key == "" { + continue + } + pol.Repo = strings.TrimSpace(pol.Repo) + m[key] = pol + } + p.referenceMu.Lock() + p.referenceRepos = m + p.referenceMu.Unlock() +} + +// referenceRepoFor returns the policy covering the repo a path names. +func (p *GitHubProxy) referenceRepoFor(path string) (ReferenceRepoPolicy, bool) { + p.referenceMu.RLock() + defer p.referenceMu.RUnlock() + if len(p.referenceRepos) == 0 { + return ReferenceRepoPolicy{}, false + } + repo := ExtractRepo(path) + if repo == "" { + repo = WriteSurfaceRepo(path) + } + if repo == "" { + return ReferenceRepoPolicy{}, false + } + pol, ok := p.referenceRepos[strings.ToLower(repo)] + return pol, ok +} + +// referenceRepoRefusal is the per-request gate the MITM loop calls. It returns +// the refusal reason and true when the request must be blocked. The own-item +// and default-branch checks consult GitHub through p.referenceLookup; a lookup +// that fails refuses (fail closed) and says so. +func (p *GitHubProxy) referenceRepoRefusal(req *http.Request) (string, bool) { + pol, ok := p.referenceRepoFor(repoScopePath(req)) + if !ok { + return "", false + } + d, number, reason := ReferenceRepoDecision(pol, req.Method, req.URL.Path, req.URL.RawQuery) + switch d { + case referenceNotReference, referenceAllow: + return "", false + case referenceDeny: + return reason, true + case referenceOwnItem: + author, err := p.referenceItemAuthor(req, pol, number) + if err != nil { + return referenceRepoRefusal(pol, fmt.Sprintf("could not confirm #%d is the hive's own item (%v)", number, err)), true + } + if pol.Login == "" || !strings.EqualFold(author, pol.Login) { + return referenceRepoRefusal(pol, fmt.Sprintf("#%d was opened by %s, not by the hive (%s); comments are allowed only on the hive's own issues and PRs", number, author, pol.Login)), true + } + return "", false + case referencePush: + if refGitInfoRefsRe.MatchString(req.URL.Path) { + return "", false // ref advertisement; the refs travel in the POST + } + def, err := p.referenceDefaultBranch(req, pol) + if err != nil { + return referenceRepoRefusal(pol, fmt.Sprintf("could not resolve its default branch to protect it (%v)", err)), true + } + refs, err := peekReceivePackRefs(req) + if err != nil { + return referenceRepoRefusal(pol, "push command list unreadable ("+err.Error()+")"), true + } + for _, ref := range refs { + if ref == "refs/heads/"+def { + return referenceRepoRefusal(pol, "pushing to its default branch ("+def+") is never allowed — push a topic branch and open a PR"), true + } + } + return "", false + } + return "", false +} + +// referenceWriteCredential says whether a request to a reference repo should +// carry the policy's write token (a write, or the push ref advertisement) — +// every other request to a reference repo goes out unauthenticated. +func referenceWriteCredential(pol ReferenceRepoPolicy, method, path, rawQuery string) bool { + if pol.Token == nil { + return false + } + d, _, _ := ReferenceRepoDecision(pol, method, path, rawQuery) + return d != referenceNotReference && d != referenceDeny +} + +// referenceAPIBase derives the REST base for the forge host a request is bound +// for: api.github.com for github.com, <host>/api/v3 for GHE. +func referenceAPIBase(req *http.Request) string { + host := canonicalHost(req.Host) + if host == "" { + host = canonicalHost(req.URL.Host) + } + switch host { + case "", "github.com", "api.github.com": + return "https://api.github.com" + } + return "https://" + host + "/api/v3" +} + +// referenceLookupFunc is the GET the own-item and default-branch checks use. +// Tests swap it; production uses an ordinary client from this process. +type referenceLookupFunc func(url, token string) ([]byte, error) + +func defaultReferenceLookup(url, token string) ([]byte, error) { + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + if token != "" { + req.Header.Set("Authorization", "token "+token) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, referenceLookupLimit)) + if err != nil { + return nil, err + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("GitHub answered %d", resp.StatusCode) + } + return body, nil +} + +func (p *GitHubProxy) referenceLookupOrDefault() referenceLookupFunc { + if p.referenceLookup != nil { + return p.referenceLookup + } + return defaultReferenceLookup +} + +func (p *GitHubProxy) referenceToken(pol ReferenceRepoPolicy) string { + if pol.Token == nil { + return "" + } + tok, ok := pol.Token() + if !ok { + return "" + } + return tok +} + +func (p *GitHubProxy) referenceItemAuthor(req *http.Request, pol ReferenceRepoPolicy, number int) (string, error) { + body, err := p.referenceLookupOrDefault()(referenceAPIBase(req)+"/repos/"+pol.Repo+"/issues/"+strconv.Itoa(number), p.referenceToken(pol)) + if err != nil { + return "", err + } + var item struct { + User struct { + Login string `json:"login"` + } `json:"user"` + } + if err := json.Unmarshal(body, &item); err != nil { + return "", err + } + if item.User.Login == "" { + return "", fmt.Errorf("item has no author") + } + return item.User.Login, nil +} + +// referenceDefaultBranchTTL bounds how long a resolved default branch is +// trusted before it is looked up again. +const referenceDefaultBranchTTL = 10 * time.Minute + +func (p *GitHubProxy) referenceDefaultBranch(req *http.Request, pol ReferenceRepoPolicy) (string, error) { + key := strings.ToLower(pol.Repo) + p.referenceMu.RLock() + cached, ok := p.referenceDefaults[key] + p.referenceMu.RUnlock() + if ok && time.Now().Before(cached.until) { + return cached.branch, nil + } + body, err := p.referenceLookupOrDefault()(referenceAPIBase(req)+"/repos/"+pol.Repo, p.referenceToken(pol)) + if err != nil { + return "", err + } + var meta struct { + DefaultBranch string `json:"default_branch"` + } + if err := json.Unmarshal(body, &meta); err != nil { + return "", err + } + if meta.DefaultBranch == "" { + return "", fmt.Errorf("repository metadata carries no default_branch") + } + p.referenceMu.Lock() + if p.referenceDefaults == nil { + p.referenceDefaults = map[string]referenceDefaultBranch{} + } + p.referenceDefaults[key] = referenceDefaultBranch{branch: meta.DefaultBranch, until: time.Now().Add(referenceDefaultBranchTTL)} + p.referenceMu.Unlock() + return meta.DefaultBranch, nil +} + +type referenceDefaultBranch struct { + branch string + until time.Time +} + +// receivePackPeekLimit bounds the command section read from a push body. The +// command list precedes the packfile and is a handful of lines; anything +// larger is not a push shape this gate will reason about. +const receivePackPeekLimit = 64 * 1024 + +// peekReceivePackRefs reads the pkt-line command section of a git-receive-pack +// body — "<old> <new> <ref>[\0caps]" lines up to the flush — returns the refs +// it names, and puts every byte back so the request forwards unchanged. A +// content-encoded body cannot be read here and is an error (fail closed). +func peekReceivePackRefs(req *http.Request) ([]string, error) { + if req.Body == nil { + return nil, fmt.Errorf("no body") + } + if enc := req.Header.Get("Content-Encoding"); enc != "" && enc != "identity" { + return nil, fmt.Errorf("content-encoding %q", enc) + } + br := bufio.NewReader(io.LimitReader(req.Body, receivePackPeekLimit)) + var consumed bytes.Buffer + tee := io.TeeReader(br, &consumed) + var refs []string + for { + var hdr [4]byte + if _, err := io.ReadFull(tee, hdr[:]); err != nil { + return nil, fmt.Errorf("pkt-line header: %w", err) + } + n, err := hex.DecodeString(string(hdr[:])) + if err != nil { + return nil, fmt.Errorf("pkt-line length %q", hdr) + } + size := int(n[0])<<8 | int(n[1]) + if size == 0 { + break // flush: end of commands + } + if size < 4 { + return nil, fmt.Errorf("pkt-line length %d", size) + } + line := make([]byte, size-4) + if _, err := io.ReadFull(tee, line); err != nil { + return nil, fmt.Errorf("pkt-line body: %w", err) + } + if i := bytes.IndexByte(line, 0); i >= 0 { + line = line[:i] + } + fields := strings.Fields(string(line)) + if len(fields) != 3 { + return nil, fmt.Errorf("malformed command %q", line) + } + refs = append(refs, fields[2]) + } + // Whatever bufio read ahead past the flush belongs to the packfile; it is + // also in neither consumed nor req.Body any more, so splice it back too. + rest, _ := br.Peek(br.Buffered()) + req.Body = io.NopCloser(io.MultiReader(bytes.NewReader(consumed.Bytes()), bytes.NewReader(append([]byte(nil), rest...)), req.Body)) + return refs, nil +} + +// referenceWriteAllowedByMode applies the agent's ACMM mode to an opt-in +// reference write: the per-repo opt-in widens WHERE an agent may write, never +// WHAT a mode lets it do. Opening an issue or commenting needs ISSUES_ONLY; +// pushing a branch or opening a PR needs ISSUES_AND_PRS, exactly as on a +// managed repo (the PR hard-deny that routes managed-repo PRs through +// hive-open-pr does not apply — the relay cannot act on a foreign repo, and +// the reference credential is the App bot's own). +func referenceWriteAllowedByMode(mode agent.AgentMode, method, path string) bool { + switch { + case refGitReceivePackRe.MatchString(path), refGitInfoRefsRe.MatchString(path), refPullsPath.MatchString(path): + return mode >= agent.ModeIssuesAndPRs + } + return mode >= agent.ModeIssuesOnly +} diff --git a/src/crux/proxy/reference_repos_test.go b/src/crux/proxy/reference_repos_test.go new file mode 100644 index 0000000000..546ed31470 --- /dev/null +++ b/src/crux/proxy/reference_repos_test.go @@ -0,0 +1,237 @@ +package proxy + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +func newReferenceTestProxy(t *testing.T, pol ReferenceRepoPolicy, lookup referenceLookupFunc) *GitHubProxy { + t.Helper() + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.SetReferenceRepos([]ReferenceRepoPolicy{pol}) + p.referenceLookup = lookup + return p +} + +func refReq(method, path, rawQuery string, body []byte) *http.Request { + req, _ := http.NewRequest(method, "https://api.github.com"+path, bytes.NewReader(body)) + req.URL.RawQuery = rawQuery + req.Host = "api.github.com" + return req +} + +// pktLine encodes one git pkt-line. +func pktLine(s string) string { return fmt.Sprintf("%04x%s", len(s)+4, s) } + +func TestReferenceRepoDecision_Matrix(t *testing.T) { + ro := ReferenceRepoPolicy{Repo: "hivecommons/docs"} + issues := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true} + prs := ReferenceRepoPolicy{Repo: "hivecommons/docs", PRs: true} + both := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, PRs: true} + + cases := []struct { + name string + pol ReferenceRepoPolicy + method string + path string + query string + want referenceDecision + }{ + // Reads pass for every tier. + {"ro GET", ro, "GET", "/repos/hivecommons/docs/contents/README.md", "", referenceNotReference}, + {"ro fetch", ro, "POST", "/hivecommons/docs.git/git-upload-pack", "", referenceNotReference}, + {"ro info/refs upload", ro, "GET", "/hivecommons/docs.git/info/refs", "service=git-upload-pack", referenceNotReference}, + // Read-only refuses every write, including the push advertisement. + {"ro issue", ro, "POST", "/repos/hivecommons/docs/issues", "", referenceDeny}, + {"ro comment", ro, "POST", "/repos/hivecommons/docs/issues/3/comments", "", referenceDeny}, + {"ro push adv", ro, "GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", referenceDeny}, + {"ro push", ro, "POST", "/hivecommons/docs.git/git-receive-pack", "", referenceDeny}, + {"ro fork", ro, "POST", "/repos/hivecommons/docs/forks", "", referenceDeny}, + // issues tier. + {"issues open", issues, "POST", "/repos/hivecommons/docs/issues", "", referenceAllow}, + {"issues comment own-check", issues, "POST", "/repos/hivecommons/docs/issues/3/comments", "", referenceOwnItem}, + {"issues pr comment own-check", issues, "POST", "/repos/hivecommons/docs/pulls/9/comments", "", referenceOwnItem}, + {"issues edit", issues, "PATCH", "/repos/hivecommons/docs/issues/3", "", referenceDeny}, + {"issues label", issues, "POST", "/repos/hivecommons/docs/issues/3/labels", "", referenceDeny}, + {"issues assign", issues, "POST", "/repos/hivecommons/docs/issues/3/assignees", "", referenceDeny}, + {"issues open pr", issues, "POST", "/repos/hivecommons/docs/pulls", "", referenceDeny}, + {"issues push", issues, "POST", "/hivecommons/docs.git/git-receive-pack", "", referenceDeny}, + // prs tier. + {"prs open pr", prs, "POST", "/repos/hivecommons/docs/pulls", "", referenceAllow}, + {"prs push adv", prs, "GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", referencePush}, + {"prs push", prs, "POST", "/hivecommons/docs.git/git-receive-pack", "", referencePush}, + {"prs open issue", prs, "POST", "/repos/hivecommons/docs/issues", "", referenceDeny}, + {"prs merge", prs, "PUT", "/repos/hivecommons/docs/pulls/9/merge", "", referenceDeny}, + {"prs review", prs, "POST", "/repos/hivecommons/docs/pulls/9/reviews", "", referenceDeny}, + {"prs edit pr", prs, "PATCH", "/repos/hivecommons/docs/pulls/9", "", referenceDeny}, + {"prs api ref", prs, "POST", "/repos/hivecommons/docs/git/refs", "", referenceDeny}, + {"prs delete ref", prs, "DELETE", "/repos/hivecommons/docs/git/refs/heads/x", "", referenceDeny}, + {"prs contents", prs, "PUT", "/repos/hivecommons/docs/contents/x.md", "", referenceDeny}, + // both. + {"both issue", both, "POST", "/repos/hivecommons/docs/issues", "", referenceAllow}, + {"both pr", both, "POST", "/repos/hivecommons/docs/pulls", "", referenceAllow}, + {"both merge", both, "PUT", "/repos/hivecommons/docs/pulls/9/merge", "", referenceDeny}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, _, reason := ReferenceRepoDecision(tc.pol, tc.method, tc.path, tc.query) + if got != tc.want { + t.Fatalf("decision=%v want %v (reason %q)", got, tc.want, reason) + } + if got == referenceDeny && (!strings.Contains(reason, "reference_repos") || !strings.Contains(reason, tc.pol.tier())) { + t.Fatalf("refusal must name reference_repos and the tier: %q", reason) + } + }) + } +} + +func TestReferenceRepoRefusal_OwnItemCheck(t *testing.T) { + pol := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, Login: "hivecommons-hive[bot]", Token: func() (string, bool) { return "ref-tok", true }} + var gotURL, gotToken string + author := "hivecommons-hive[bot]" + p := newReferenceTestProxy(t, pol, func(url, token string) ([]byte, error) { + gotURL, gotToken = url, token + return []byte(`{"user":{"login":"` + author + `"}}`), nil + }) + + if reason, refused := p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil)); refused { + t.Fatalf("comment on own issue refused: %s", reason) + } + if gotURL != "https://api.github.com/repos/hivecommons/docs/issues/12" || gotToken != "ref-tok" { + t.Fatalf("lookup url=%q token=%q", gotURL, gotToken) + } + + author = "someone-else" + reason, refused := p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil)) + if !refused || !strings.Contains(reason, "someone-else") { + t.Fatalf("comment on another's issue must be refused naming the author, got refused=%v %q", refused, reason) + } + + p.referenceLookup = func(string, string) ([]byte, error) { return nil, fmt.Errorf("boom") } + if _, refused := p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil)); !refused { + t.Fatal("a failed author lookup must fail closed") + } + + // A different repo is not a reference repo at all. + if _, refused := p.referenceRepoRefusal(refReq("POST", "/repos/cncf/techdocs/issues/1/comments", "", nil)); refused { + t.Fatal("non-reference repo must not be gated here") + } +} + +func TestReferenceRepoRefusal_PushGuardsDefaultBranch(t *testing.T) { + pol := ReferenceRepoPolicy{Repo: "hivecommons/docs", PRs: true, Token: func() (string, bool) { return "ref-tok", true }} + lookups := 0 + p := newReferenceTestProxy(t, pol, func(url, token string) ([]byte, error) { + lookups++ + if url != "https://api.github.com/repos/hivecommons/docs" { + t.Fatalf("unexpected lookup %s", url) + } + return []byte(`{"default_branch":"main"}`), nil + }) + zero := strings.Repeat("0", 40) + sha := strings.Repeat("a", 40) + pack := "PACK-bytes-follow" + + topic := pktLine(zero+" "+sha+" refs/heads/hive/assessment\x00report-status side-band-64k") + "0000" + pack + req := refReq("POST", "/hivecommons/docs.git/git-receive-pack", "", []byte(topic)) + if reason, refused := p.referenceRepoRefusal(req); refused { + t.Fatalf("topic-branch push refused: %s", reason) + } + rest, _ := io.ReadAll(req.Body) + if string(rest) != topic { + t.Fatalf("push body not restored byte-for-byte:\n got %q\nwant %q", rest, topic) + } + + def := pktLine(zero+" "+sha+" refs/heads/hive/x\x00caps") + pktLine(sha+" "+sha+" refs/heads/main") + "0000" + pack + reason, refused := p.referenceRepoRefusal(refReq("POST", "/hivecommons/docs.git/git-receive-pack", "", []byte(def))) + if !refused || !strings.Contains(reason, "default branch (main)") { + t.Fatalf("default-branch push must be refused, got refused=%v %q", refused, reason) + } + if lookups != 1 { + t.Fatalf("default branch should be cached after one lookup, got %d", lookups) + } + + gz := refReq("POST", "/hivecommons/docs.git/git-receive-pack", "", []byte(topic)) + gz.Header.Set("Content-Encoding", "gzip") + if _, refused := p.referenceRepoRefusal(gz); !refused { + t.Fatal("an encoded push body the guard cannot read must fail closed") + } + + if _, refused := p.referenceRepoRefusal(refReq("GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", nil)); refused { + t.Fatal("push ref advertisement must pass for a prs entry") + } +} + +func TestReferenceRepoAuth_StripsOnReadsInjectsOnWrites(t *testing.T) { + pol := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, PRs: true, Token: func() (string, bool) { return "ref-tok", true }} + p := newReferenceTestProxy(t, pol, nil) + p.injectGHAuth = true + p.agentTokenSource = func(string) (string, bool) { return "hive-app-token", true } + + read := refReq("GET", "/repos/hivecommons/docs/contents/README.md", "", nil) + read.Header.Set("Authorization", "token agent-held") + p.rewriteGitHubAuth(read, "scanner") + if got := read.Header.Get("Authorization"); got != "" { + t.Fatalf("reference read must go out unauthenticated, got %q", got) + } + + fetch := refReq("POST", "/hivecommons/docs.git/git-upload-pack", "", nil) + fetch.Header.Set("Authorization", "Basic xyz") + p.rewriteGitHubAuth(fetch, "scanner") + if got := fetch.Header.Get("Authorization"); got != "" { + t.Fatalf("reference fetch must go out unauthenticated, got %q", got) + } + + write := refReq("POST", "/repos/hivecommons/docs/issues", "", nil) + write.Header.Set("Authorization", "token agent-held") + p.rewriteGitHubAuth(write, "scanner") + if got := write.Header.Get("Authorization"); got != "token ref-tok" { + t.Fatalf("reference write must carry the reference credential, got %q", got) + } + + push := refReq("GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", nil) + p.rewriteGitHubAuth(push, "scanner") + want := "Basic " + base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser+":ref-tok")) + if got := push.Header.Get("Authorization"); got != want || !push.Close { + t.Fatalf("push advertisement must carry basic reference credential and close, got %q close=%v", got, push.Close) + } + + // A managed repo still takes the ordinary injection path. + managed := refReq("GET", "/repos/cncf/techdocs/issues", "", nil) + p.rewriteGitHubAuth(managed, "scanner") + if got := managed.Header.Get("Authorization"); got != "token hive-app-token" { + t.Fatalf("managed repo should get the agent token, got %q", got) + } + + // Read-only entries never carry anything, even on a (refused upstream) write. + p.SetReferenceRepos([]ReferenceRepoPolicy{{Repo: "hivecommons/docs"}}) + ro := refReq("POST", "/repos/hivecommons/docs/issues", "", nil) + ro.Header.Set("Authorization", "token agent-held") + p.rewriteGitHubAuth(ro, "scanner") + if got := ro.Header.Get("Authorization"); got != "" { + t.Fatalf("read-only reference repo must never carry a credential, got %q", got) + } +} + +func TestReferenceWriteAllowedByMode(t *testing.T) { + if referenceWriteAllowedByMode(agent.ModeAdvisory, "POST", "/repos/o/r/issues") { + t.Error("ADVISORY may not open issues on a reference repo") + } + if !referenceWriteAllowedByMode(agent.ModeIssuesOnly, "POST", "/repos/o/r/issues") { + t.Error("ISSUES_ONLY may open issues") + } + if referenceWriteAllowedByMode(agent.ModeIssuesOnly, "POST", "/repos/o/r/pulls") { + t.Error("ISSUES_ONLY may not open PRs") + } + if !referenceWriteAllowedByMode(agent.ModeIssuesAndPRs, "POST", "/o/r.git/git-receive-pack") { + t.Error("ISSUES_AND_PRS may push a topic branch") + } +} diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 83820127f5..07ab917f90 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1693` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1694` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1695` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1704` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1705` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1706` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 96f6d9c44f..dc4eacc334 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1700` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1711` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index acefbb9f3c..f26adaa4ae 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1673` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1674` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1675` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1686` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1687` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1688` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1689` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1690` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1691` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1692` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1684` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1685` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1686` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1692` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1697` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1698` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1699` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1700` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1701` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1702` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1703` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index d6909a3512..8dde400125 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1763` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1774` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index bd4a8b6d1e..61ebcbd711 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -252,6 +252,21 @@ A refused request is handled like every other policy denial: Refusals are not counted as output activity. +## Reference repos + +`project.reference_repos` ([#18015](https://github.com/hivecommons/hive/issues/18015)) +names repos *outside* the project an agent may study. They are not part of +the write surface at all by default: the proxy strips the hive's App token +from every request to them and refuses every non-GET with a message naming +`reference_repos`. A per-entry `write: [issues, prs]` opt-in unlocks exactly +three direct writes on that repo — open an issue, open a PR / push a topic +branch, comment on the hive's own issue or PR — sent with a credential valid +for *that* repo (the App installed on the target org, or `write_token_file`). +Merge, default-branch push, labels, assignees, reviews and closing others' +items stay refused. The relays never act on a reference repo, so a lane under +`write_surface.enforce` cannot write to one. See the +[operator reference](operator-reference/README.md#configuration-blocks). + ## Enforcing the write surface `write_surface.enforce` lists the lanes whose **direct** GitHub writes from the diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 35267a3a32..48998e010e 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -251,6 +251,9 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `project.repo_policies[].label_driven` | Off (unset) for every repo. | Opt-in for repos whose maintainers accept and park issues with labels and their own bots (for example `needs-triage` → `triage/accepted`). On such a repo the un-park sweep posts no "What to reply" notice and never removes `needs-human`, `needs-decision` or `needs-direction` on a `/hive` comment alone; instead a trusted triager applying an approval label releases the issue and counts as human sign-off — see [Label-driven approval](#label-driven-approval). Editable from the dashboard: Settings → Repos → per-repo "Label-driven approvals (Prow-style)" (write or owner role). Details in [maintainer-commands.md](../maintainer-commands.md#label-driven-repositories). | | `project.repo_policies[].label_driven_approval_labels` | Unset: inherits `project.label_driven_approval_labels`. | Per-repo approval-label list; only read when `label_driven` is set. Editable from the dashboard sub-row of the repo's label-driven toggle. | | `project.repo_policies[].companion_repos` | Unset (no companions). | List of sibling repositories (`name` in the project org, or `org/name`) that this repo's tooling expects checked out beside it under the agent workspace root — for example a docs/website repo a spec tool registers at `<workspace>/<org>/<sibling>/…`. Contributor task prompts and resident-agent kicks carry a shallow-clone plan for each companion at `<workspace root>/<org>/<sibling>` (`$HIVE_WORKSPACE_DIR` for contributor agents, `$HOME/workspace` for resident agents), and agents are told that a tool reporting one of those paths missing means *clone it*, never park the issue. Self-references and malformed entries are dropped. See [running-at-level-6.md](../running-at-level-6.md#needless-parks). | +| `project.reference_repos` | Unset (none). | Repos **outside** the project the hive may *study* without managing them ([#18015](https://github.com/hivecommons/hive/issues/18015)) — the rare case of a hive whose deliverable is *about* another project, e.g. a CNCF TechDocs hive (authorized repo `cncf/techdocs`) assessing `hivecommons/docs`. Each entry is `owner/name` (bare names are rejected, as is any repo also in `project.repos`). Default tier is **read-only**: agents may `git clone`/fetch them unauthenticated over HTTPS and issue GitHub REST/GraphQL GETs; the proxy strips the hive's App token from every request to them and refuses every non-GET with a message naming `reference_repos`. The kick lists them under `📚 REFERENCE REPOS` so an authorized-repo item whose deliverables name one is treated as in scope (not out-of-scope/critical), and agents are told findings go in the deliverable inside the authorized repo, never as issues or PRs on the reference repo. The dashboard shows each as a minimal card with a `reference · <tier>` chip — no stats, no ACMM, no work items. Private reference repos are not readable (reads are unauthenticated). Example: <br>`project:`<br>` repos: [techdocs]`<br>` reference_repos:`<br>` - hivecommons/docs`<br>` - hivecommons/hivecommons.github.io`<br>` - {repo: hivecommons/hive, write: [issues, prs]}` | +| `project.reference_repos[].write` | Unset (read-only). | Per-entry write opt-in, any of `issues`, `prs`. The **only** writes it unlocks on that repo: open an issue; open a PR / push a topic branch; comment on an issue or PR the hive itself opened. Never: merge, push to the default branch, labels or assignees on others' items, reviews, closing others' issues — the proxy refuses those deterministically (`reference_repo` in the violations log). The write is sent with a credential GitHub honours for *that* repo, never the hive's own installation token: by default the same GitHub App must be **installed on the target org** (the hive discovers that installation and mints a token scoped to the one repo and the opted-in permissions); otherwise set `write_token_file`. Boot refuses (exit code 22) a write-mode entry with neither. Direct POSTs are used here, not the `hive-open-pr`/`hive-open-issue` relays, which only act on `project.repos`; a lane under `write_surface.enforce` therefore cannot write to a reference repo. | +| `project.reference_repos[].write_token_file` | Unset. | Path to a file holding a token (a maintainer PAT or fine-grained token) authorized for that repo; read on every use so a rotation needs no restart. Only meaningful with `write`. Its `GET /user` login is what the proxy treats as "own item" for comments. | | `project.label_driven_approval_labels` | `["triage/accepted"]` | Labels whose trusted addition approves an issue on every label-driven repo without its own list. Ignored by repos without `label_driven: true`. Dashboard: Settings → Repos → "Default approval labels". | | `project.label_driven_trust_org_members` | `true` | Whether a public member of the repo's owning org who applies an approval label is trusted (Prow's own rule). Dashboard: Settings → Repos → "trust public org members". | | `project.label_driven_trusted_bots` | `["kubestellar-prow[bot]"]` | Bot logins whose label additions count as a trusted triager (Prow command bots relaying `/triage accepted`). `github-actions[bot]` is not in the default because it is the shared identity of every workflow in the repo, including ones triggered by untrusted events; operators whose Prow emulation runs as `github-actions[bot]` must add it explicitly per deployment. Dashboard: Settings → Repos → "Trusted bots". | diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index ecf4cb63f0..8b54b34f80 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1168,6 +1168,13 @@ type ProjectConfig struct { // Optional and additive. Absent — the state of every existing config — // means nothing is paused and the hive behaves exactly as before. PausedRepos []RepoPause `yaml:"paused_repos,omitempty"` + // ReferenceRepos are repositories agents may STUDY but do not manage + // (hivecommons/hive#18015): explicit owner/name entries, read-only by + // default, with a rare per-entry opt-in for opening issues/PRs. They are + // NOT in Repos: no work is enumerated from them, no ACMM eval runs on + // them, and writes to them are refused unless opted in. See + // reference_repos.go. + ReferenceRepos []ReferenceRepo `yaml:"reference_repos,omitempty" json:"reference_repos,omitempty"` // RepoPolicies stores optional per-repository policy overrides keyed by // repo name. project.repos remains the watched-repo identity list; this // sidecar list lets existing string-list configs keep round-tripping. diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go new file mode 100644 index 0000000000..f67a976c34 --- /dev/null +++ b/src/pkg/config/reference_repos.go @@ -0,0 +1,231 @@ +package config + +import ( + "fmt" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// ReferenceRepo is one entry of project.reference_repos (hivecommons/hive#18015): +// a repository the hive's agents may STUDY — clone, fetch, read through the +// GitHub API — without it being an authorized repo. The motivating case is a +// documentation-assessment hive (CNCF TechDocs) whose deliverable lives in its +// own authorized repo but describes another project's repos: the agent has to +// read those repos, and must not be told they are out of scope, yet nothing +// about the assessment should ever land on them. +// +// The default tier is READ-ONLY: public clones and GETs, never carrying the +// hive's App token (which is not valid for a foreign installation anyway). +// Every write is refused by the proxy with a 403 that names reference_repos. +// +// Write is the rare opt-in for an assessor who also files findings upstream. +// It is deliberately narrow — open an issue, open a PR on a topic branch, +// comment on the hive's own items — and the credential behind it has to be one +// GitHub actually honours for that repo: the App's installation on the TARGET +// org (minted on demand, scoped to this one repo) or an operator-supplied +// write_token_file. Boot refuses a write-mode entry that has neither. +type ReferenceRepo struct { + // Repo is the explicit "owner/name" reference. Bare names are rejected: + // a reference repo is by definition outside the project's own list, so + // there is no org to qualify it against without guessing. + Repo string `yaml:"repo" json:"repo"` + // Write lists the opt-in write surfaces: "issues" and/or "prs". Empty is + // the read-only default. + Write []string `yaml:"write,omitempty" json:"write,omitempty"` + // WriteTokenFile is a path on the hive host holding a token that is + // authorized for Repo, used instead of minting from the App installation + // on the target org. Only meaningful with Write. + WriteTokenFile string `yaml:"write_token_file,omitempty" json:"write_token_file,omitempty"` +} + +const ( + ReferenceWriteIssues = "issues" + ReferenceWritePRs = "prs" +) + +// UnmarshalYAML accepts both the scalar form ("owner/name", read-only) and the +// mapping form ({repo: owner/name, write: [...]}). +func (r *ReferenceRepo) UnmarshalYAML(value *yaml.Node) error { + if value.Kind == yaml.ScalarNode { + r.Repo = strings.TrimSpace(value.Value) + r.Write = nil + r.WriteTokenFile = "" + return nil + } + type plain ReferenceRepo + var p plain + if err := value.Decode(&p); err != nil { + return err + } + *r = ReferenceRepo(p) + r.Repo = strings.TrimSpace(r.Repo) + return nil +} + +// MarshalYAML writes the scalar form back for a read-only entry so a config +// that was written as a plain list round-trips unchanged. +func (r ReferenceRepo) MarshalYAML() (any, error) { + if len(r.Write) == 0 && r.WriteTokenFile == "" { + return r.Repo, nil + } + type plain ReferenceRepo + return plain(r), nil +} + +// WritesIssues reports whether the entry opted in to opening issues (and +// commenting on its own). +func (r ReferenceRepo) WritesIssues() bool { return r.hasWrite(ReferenceWriteIssues) } + +// WritesPRs reports whether the entry opted in to pushing topic branches and +// opening PRs. +func (r ReferenceRepo) WritesPRs() bool { return r.hasWrite(ReferenceWritePRs) } + +// ReadOnly reports whether the entry has no write opt-in at all. +func (r ReferenceRepo) ReadOnly() bool { return !r.WritesIssues() && !r.WritesPRs() } + +func (r ReferenceRepo) hasWrite(kind string) bool { + for _, w := range r.Write { + if strings.EqualFold(strings.TrimSpace(w), kind) { + return true + } + } + return false +} + +// WriteLabel is the human-facing tier: "read-only", "issues", "prs" or +// "issues+prs". The dashboard chip and the kick both print it. +func (r ReferenceRepo) WriteLabel() string { + switch { + case r.WritesIssues() && r.WritesPRs(): + return "issues+prs" + case r.WritesIssues(): + return "issues" + case r.WritesPRs(): + return "prs" + } + return "read-only" +} + +// Owner returns the owner half of Repo ("" when malformed). +func (r ReferenceRepo) Owner() string { + owner, _, _ := strings.Cut(r.Repo, "/") + return owner +} + +// Name returns the repository half of Repo ("" when malformed). +func (r ReferenceRepo) Name() string { + _, name, _ := strings.Cut(r.Repo, "/") + return name +} + +// referenceRepoKey is the case-folded comparison key for a reference entry. +func referenceRepoKey(repo string) string { + return strings.ToLower(strings.TrimSpace(repo)) +} + +// ReferenceRepoFor returns the reference entry covering repo ("owner/name", +// case-insensitive), and whether one exists. +func (c *Config) ReferenceRepoFor(repo string) (ReferenceRepo, bool) { + if c == nil { + return ReferenceRepo{}, false + } + key := referenceRepoKey(repo) + if key == "" { + return ReferenceRepo{}, false + } + for _, r := range c.Project.ReferenceRepos { + if referenceRepoKey(r.Repo) == key { + return r, true + } + } + return ReferenceRepo{}, false +} + +// IsReferenceRepo reports whether repo ("owner/name") is a reference repo. +func (c *Config) IsReferenceRepo(repo string) bool { + _, ok := c.ReferenceRepoFor(repo) + return ok +} + +// ReferenceReposWithWrite returns the entries that opted in to any write. +func (c *Config) ReferenceReposWithWrite() []ReferenceRepo { + if c == nil { + return nil + } + var out []ReferenceRepo + for _, r := range c.Project.ReferenceRepos { + if !r.ReadOnly() { + out = append(out, r) + } + } + return out +} + +// validReferenceRepoSlug accepts exactly "owner/name" with no whitespace, no +// scheme and no extra path. +func validReferenceRepoSlug(repo string) bool { + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" { + return false + } + if strings.ContainsAny(repo, " \t\n:@") || strings.Contains(name, "/") { + return false + } + return true +} + +// validateReferenceRepos enforces the shape of project.reference_repos: every +// entry is an explicit owner/name, none duplicates another, none is also an +// authorized repo (a repo is either managed or reference, never both — the +// two tiers would otherwise contradict each other on every write), write +// kinds are known, and write_token_file is only set alongside write. +func (c *Config) validateReferenceRepos() error { + if len(c.Project.ReferenceRepos) == 0 { + return nil + } + authorized := make(map[string]bool, len(c.Project.Repos)) + for _, repo := range c.Project.Repos { + authorized[referenceRepoKey(QualifyRepo(c.Project.Org, repo))] = true + } + seen := make(map[string]bool, len(c.Project.ReferenceRepos)) + for i, r := range c.Project.ReferenceRepos { + if !validReferenceRepoSlug(r.Repo) { + return fmt.Errorf("project.reference_repos[%d]: %q must be an explicit owner/name (a reference repo lives outside this project, so there is no org to qualify a bare name against)", i, r.Repo) + } + key := referenceRepoKey(r.Repo) + if authorized[key] { + return fmt.Errorf("project.reference_repos[%d]: %q is already in project.repos — a repository is either authorized (agents work it) or reference (agents only study it), not both", i, r.Repo) + } + if seen[key] { + return fmt.Errorf("project.reference_repos[%d]: %q is listed twice", i, r.Repo) + } + seen[key] = true + for _, w := range r.Write { + switch strings.ToLower(strings.TrimSpace(w)) { + case ReferenceWriteIssues, ReferenceWritePRs: + default: + return fmt.Errorf("project.reference_repos[%d]: write %q is not one of %q, %q", i, w, ReferenceWriteIssues, ReferenceWritePRs) + } + } + if r.WriteTokenFile != "" && r.ReadOnly() { + return fmt.Errorf("project.reference_repos[%d]: write_token_file is set but write is empty — a read-only reference repo never carries a credential", i) + } + } + return nil +} + +// ReferenceRepoSlugs returns the reference repos as sorted "owner/name" +// strings, for kicks and environment. +func (c *Config) ReferenceRepoSlugs() []string { + if c == nil { + return nil + } + out := make([]string, 0, len(c.Project.ReferenceRepos)) + for _, r := range c.Project.ReferenceRepos { + out = append(out, r.Repo) + } + sort.Strings(out) + return out +} diff --git a/src/pkg/config/reference_repos_test.go b/src/pkg/config/reference_repos_test.go new file mode 100644 index 0000000000..f0973865ec --- /dev/null +++ b/src/pkg/config/reference_repos_test.go @@ -0,0 +1,101 @@ +package config + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func referenceRepoTestConfig(t *testing.T, refs string) *Config { + t.Helper() + src := ` +project: + org: cncf + repos: [techdocs] + reference_repos: +` + refs + ` +agents: + scanner: + backend: claude +` + var c Config + if err := yaml.Unmarshal([]byte(src), &c); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return &c +} + +func TestReferenceReposParseScalarAndMapping(t *testing.T) { + c := referenceRepoTestConfig(t, ` + - hivecommons/docs + - repo: hivecommons/hivecommons.github.io + write: [issues, prs] + - repo: hivecommons/hive + write: [issues] + write_token_file: /data/ref-token +`) + if err := c.validateReferenceRepos(); err != nil { + t.Fatalf("validate: %v", err) + } + if got := len(c.Project.ReferenceRepos); got != 3 { + t.Fatalf("want 3 entries, got %d", got) + } + ro, ok := c.ReferenceRepoFor("HiveCommons/Docs") + if !ok || !ro.ReadOnly() || ro.WriteLabel() != "read-only" { + t.Fatalf("docs: ok=%v entry=%+v", ok, ro) + } + rw, ok := c.ReferenceRepoFor("hivecommons/hivecommons.github.io") + if !ok || !rw.WritesIssues() || !rw.WritesPRs() || rw.WriteLabel() != "issues+prs" { + t.Fatalf("site: ok=%v entry=%+v", ok, rw) + } + tok, _ := c.ReferenceRepoFor("hivecommons/hive") + if tok.WriteTokenFile != "/data/ref-token" || tok.WriteLabel() != "issues" || tok.Owner() != "hivecommons" || tok.Name() != "hive" { + t.Fatalf("hive: %+v", tok) + } + if c.IsReferenceRepo("cncf/techdocs") { + t.Fatal("authorized repo must not read as reference") + } + if got := len(c.ReferenceReposWithWrite()); got != 2 { + t.Fatalf("want 2 write entries, got %d", got) + } + // Round-trip: the scalar entry stays a scalar. + out, err := yaml.Marshal(c.Project.ReferenceRepos) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "- hivecommons/docs\n") { + t.Fatalf("read-only entry should marshal as a scalar:\n%s", out) + } +} + +func TestReferenceReposValidationRejects(t *testing.T) { + cases := []struct { + name, refs, want string + }{ + {"bare name", " - docs\n", "explicit owner/name"}, + {"url", " - https://github.com/hivecommons/docs\n", "explicit owner/name"}, + {"overlap with project.repos", " - cncf/techdocs\n", "already in project.repos"}, + {"overlap case-insensitive", " - CNCF/TechDocs\n", "already in project.repos"}, + {"duplicate", " - hivecommons/docs\n - HiveCommons/docs\n", "listed twice"}, + {"unknown write", " - repo: hivecommons/docs\n write: [merge]\n", "not one of"}, + {"token without write", " - repo: hivecommons/docs\n write_token_file: /x\n", "write is empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := referenceRepoTestConfig(t, tc.refs) + err := c.validateReferenceRepos() + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("want error containing %q, got %v", tc.want, err) + } + }) + } +} + +func TestReferenceReposValidationRunsFromValidate(t *testing.T) { + c := referenceRepoTestConfig(t, " - docs\n") + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "reference_repos") { + t.Fatalf("Validate should surface the reference_repos error, got %v", err) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8812f3e6cf..1cc6845b18 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -99,6 +99,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := validateWorkflowBindings(c.Project.WorkflowBindings); err != nil { return err } + if err := c.validateReferenceRepos(); err != nil { + return err + } if _, err := c.Ioscan.Policy(); err != nil { return err } diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index d53f4cc24b..a8eb8e7c30 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -657,11 +657,15 @@ type StatusPayload struct { Governor FrontendGovernor `json:"governor"` Tokens FrontendTokens `json:"tokens"` Repos []FrontendRepo `json:"repos"` - Swarm SwarmStatus `json:"swarm"` - Beads FrontendBeads `json:"beads"` - Planning FrontendPlanning `json:"planning"` - Runs RunsSummary `json:"runs"` - Health map[string]any `json:"health"` + // ReferenceRepos are project.reference_repos (hivecommons/hive#18015): + // studied, not worked. Kept apart from Repos so no queue, ACMM or + // work-item consumer ever treats one as managed. + ReferenceRepos []FrontendReferenceRepo `json:"referenceRepos,omitempty"` + Swarm SwarmStatus `json:"swarm"` + Beads FrontendBeads `json:"beads"` + Planning FrontendPlanning `json:"planning"` + Runs RunsSummary `json:"runs"` + Health map[string]any `json:"health"` // DeepHealth carries the spoke's own deep health checks (HealthSummary: // ready, github_auth, agents, …) — the same checks the heartbeat reports // to the hub. The dashboard's header Health pill renders from these, NOT @@ -1211,6 +1215,13 @@ type FrontendRestartRefresh struct { ReposTotal int `json:"reposTotal"` } +// FrontendReferenceRepo is one project.reference_repos entry for the repos +// card: the slug and its tier ("read-only", "issues", "prs", "issues+prs"). +type FrontendReferenceRepo struct { + Full string `json:"full"` + Tier string `json:"tier"` +} + type FrontendRepo struct { Name string `json:"name"` Full string `json:"full"` diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 3bf0e75a3c..6c298855c7 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -482,6 +482,9 @@ project.paused_repos[].by # TODO #11854 project.paused_repos[].reason # TODO #11854 project.paused_repos[].repo # TODO #11854 project.primary_repo # TODO #11854 +project.reference_repos[].repo # file-only by design (#18015): rare study-only repos, set in the overlay +project.reference_repos[].write[] # file-only by design (#18015) +project.reference_repos[].write_token_file # file-only by design (#18015) project.repo_policies[].acmm_last_automatic.at # TODO #11854 project.repo_policies[].acmm_last_automatic.direction # TODO #11854 project.repo_policies[].acmm_last_automatic.evidence_ids[] # TODO #11854 diff --git a/src/pkg/dashboard/static-src/css/repos.css b/src/pkg/dashboard/static-src/css/repos.css index cb28002577..c8aa80659c 100644 --- a/src/pkg/dashboard/static-src/css/repos.css +++ b/src/pkg/dashboard/static-src/css/repos.css @@ -73,6 +73,13 @@ .repo-fork-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--blue) 45%, transparent); background: color-mix(in srgb, var(--blue) 15%, transparent); color: var(--blue); cursor: help; } .repo-fork-pill.off { border-color: var(--border); background: transparent; color: var(--text-muted); } .repo-paused-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; text-transform: uppercase; border: 1px solid color-mix(in srgb, var(--yellow) 45%, transparent); background: color-mix(in srgb, var(--yellow) 15%, transparent); color: var(--yellow); cursor: help; } + /* project.reference_repos (#18015): a studied-not-worked repo. Muted + border, no stats block, one tier chip. */ + .repo-card-reference { border-style: dashed; opacity: 0.85; } + .repo-reference-provider { color: var(--muted); } + .repo-reference-link { color: inherit; text-decoration: none; } + .repo-reference-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--indigo) 45%, transparent); background: color-mix(in srgb, var(--indigo) 12%, transparent); color: var(--indigo); cursor: help; } + .repo-reference-note { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); } .repo-pause-btn { flex: 0 0 auto; margin-left: 0; font-size: 0.66rem; font-weight: 600; line-height: 1; padding: 3px 7px; border-radius: var(--r-pill); white-space: nowrap; cursor: pointer; border: 1px solid var(--border); background: transparent; color: var(--muted); opacity: 0.7; transition: opacity 0.15s, color 0.15s, border-color 0.15s; } .repo-automerge-toggle { flex: 0 0 auto; white-space: nowrap; gap: var(--sp-2); margin: 0; font-size: var(--fs-2xs); color: var(--muted); } /* margin: 0 beats .config-toggle's margin-bottom (that rule is declared diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index c60dde40c0..c8f98b6833 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -346,12 +346,37 @@ <div class="repo-stat"><a href="${esc(repoUrl + cardForge.changesPath)}" target="_blank" rel="noopener" style="color:inherit;text-decoration:none"><div class="spark-row"><span class="num">${r.prs >= 0 ? r.prs : '?'}</span>${pSpark}</div><div class="label">PRs</div></a>${prBreakdown}</div> </div>${pillsHtml}${hfRepoSectionHtml(r)} </div>`; - }).join('')); + }).join('') + renderReferenceRepoCards(status.referenceRepos, hostLabel)); hfRenderEditor(); hfRenderPinned(); hfRenderCrossRepo(); } + /** + * project.reference_repos (#18015): repos outside the project the hive may + * study. Deliberately minimal — a name and a tier chip, no stats, no ACMM, + * no pause/swarm controls — so they are legible as "not managed here". + * Hidden while a pill or project filter is active (they have no work items + * to match). + */ + function renderReferenceRepoCards(refs, hostLabel) { + if (!Array.isArray(refs) || !refs.length) return ''; + if (repoPillFilterActive(repoPillFilterState()) || overviewProjectFilterState()) return ''; + const base = window._githubBaseUrl || 'https://github.com'; + return refs.map(rr => { + const full = String(rr.full || ''); + const tier = String(rr.tier || 'read-only'); + const tip = tier === 'read-only' + ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' + : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; + return `<div class="repo-card repo-card-reference" data-repo="${esc(full)}"> + <div class="repo-provider-label repo-reference-provider" title="This repo lives on ${esc(hostLabel)} and is outside this project">${esc(hostLabel)}</div> + <div class="repo-name"><a class="repo-reference-link" href="${esc(base + '/' + full)}" target="_blank" rel="noopener" title="${esc(full)}">${esc(full)}</a><span class="repo-header-badges"><span class="repo-reference-pill" title="${esc(tip)}">📚 reference · ${esc(tier)}</span></span></div> + <div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> + </div>`; + }).join(''); + } + /** * Where a repo card points and what it calls its host (#14115). r.forge is * the repo's forge kind from /api/status; GitHub (and an older hive that diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 22024fe971..9ed7f34a33 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -3299,6 +3299,13 @@ .repo-fork-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--blue) 45%, transparent); background: color-mix(in srgb, var(--blue) 15%, transparent); color: var(--blue); cursor: help; } .repo-fork-pill.off { border-color: var(--border); background: transparent; color: var(--text-muted); } .repo-paused-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; text-transform: uppercase; border: 1px solid color-mix(in srgb, var(--yellow) 45%, transparent); background: color-mix(in srgb, var(--yellow) 15%, transparent); color: var(--yellow); cursor: help; } + /* project.reference_repos (#18015): a studied-not-worked repo. Muted + border, no stats block, one tier chip. */ + .repo-card-reference { border-style: dashed; opacity: 0.85; } + .repo-reference-provider { color: var(--muted); } + .repo-reference-link { color: inherit; text-decoration: none; } + .repo-reference-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--indigo) 45%, transparent); background: color-mix(in srgb, var(--indigo) 12%, transparent); color: var(--indigo); cursor: help; } + .repo-reference-note { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); } .repo-pause-btn { flex: 0 0 auto; margin-left: 0; font-size: 0.66rem; font-weight: 600; line-height: 1; padding: 3px 7px; border-radius: var(--r-pill); white-space: nowrap; cursor: pointer; border: 1px solid var(--border); background: transparent; color: var(--muted); opacity: 0.7; transition: opacity 0.15s, color 0.15s, border-color 0.15s; } .repo-automerge-toggle { flex: 0 0 auto; white-space: nowrap; gap: var(--sp-2); margin: 0; font-size: var(--fs-2xs); color: var(--muted); } /* margin: 0 beats .config-toggle's margin-bottom (that rule is declared @@ -36875,12 +36882,37 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> <div class="repo-stat"><a href="${esc(repoUrl + cardForge.changesPath)}" target="_blank" rel="noopener" style="color:inherit;text-decoration:none"><div class="spark-row"><span class="num">${r.prs >= 0 ? r.prs : '?'}</span>${pSpark}</div><div class="label">PRs</div></a>${prBreakdown}</div> </div>${pillsHtml}${hfRepoSectionHtml(r)} </div>`; - }).join('')); + }).join('') + renderReferenceRepoCards(status.referenceRepos, hostLabel)); hfRenderEditor(); hfRenderPinned(); hfRenderCrossRepo(); } + /** + * project.reference_repos (#18015): repos outside the project the hive may + * study. Deliberately minimal — a name and a tier chip, no stats, no ACMM, + * no pause/swarm controls — so they are legible as "not managed here". + * Hidden while a pill or project filter is active (they have no work items + * to match). + */ + function renderReferenceRepoCards(refs, hostLabel) { + if (!Array.isArray(refs) || !refs.length) return ''; + if (repoPillFilterActive(repoPillFilterState()) || overviewProjectFilterState()) return ''; + const base = window._githubBaseUrl || 'https://github.com'; + return refs.map(rr => { + const full = String(rr.full || ''); + const tier = String(rr.tier || 'read-only'); + const tip = tier === 'read-only' + ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' + : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; + return `<div class="repo-card repo-card-reference" data-repo="${esc(full)}"> + <div class="repo-provider-label repo-reference-provider" title="This repo lives on ${esc(hostLabel)} and is outside this project">${esc(hostLabel)}</div> + <div class="repo-name"><a class="repo-reference-link" href="${esc(base + '/' + full)}" target="_blank" rel="noopener" title="${esc(full)}">${esc(full)}</a><span class="repo-header-badges"><span class="repo-reference-pill" title="${esc(tip)}">📚 reference · ${esc(tier)}</span></span></div> + <div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> + </div>`; + }).join(''); + } + /** * Where a repo card points and what it calls its host (#14115). r.forge is * the repo's forge kind from /api/status; GitHub (and an older hive that diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index e85b6beac4..40e031e0b0 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -404,6 +404,7 @@ func BuildFrontendStatus( Governor: buildGovernor(govState, cfg, apiBudgetFromRateLimits(ghRateLimits)), Tokens: buildTokens(tokenCollector), Repos: repos, + ReferenceRepos: buildReferenceRepos(cfg), OverviewKPIIncomplete: overviewKPIIncomplete(cfg, actionable) || overviewStaleSince(repos) != "", StaleSince: earliestRFC3339(staleSinceRFC3339(actionable), overviewStaleSince(repos)), RestartRefresh: restartRefreshInfo(actionable, repos), @@ -2372,6 +2373,19 @@ func buildRepos(cfg *config.Config, actionable *github.ActionableResult, govStat return buildReposWithForge(cfg, actionable, govState, nil) } +// buildReferenceRepos lists project.reference_repos with their tier for the +// dashboard's minimal reference cards (hivecommons/hive#18015). +func buildReferenceRepos(cfg *config.Config) []FrontendReferenceRepo { + if cfg == nil || len(cfg.Project.ReferenceRepos) == 0 { + return nil + } + out := make([]FrontendReferenceRepo, 0, len(cfg.Project.ReferenceRepos)) + for _, rr := range cfg.Project.ReferenceRepos { + out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: rr.WriteLabel()}) + } + return out +} + func buildReposWithForge(cfg *config.Config, actionable *github.ActionableResult, govState governor.State, forge forgeOpenCounter) []FrontendRepo { repos := make([]FrontendRepo, 0, len(cfg.Project.Repos)) repoRows := make(map[string]string, len(cfg.Project.Repos)*2) diff --git a/src/pkg/github/reference_repo_token.go b/src/pkg/github/reference_repo_token.go new file mode 100644 index 0000000000..7000ddfc47 --- /dev/null +++ b/src/pkg/github/reference_repo_token.go @@ -0,0 +1,190 @@ +package github + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" + + "github.com/hivecommons/hive/pkg/config" +) + +// Reference-repo write credentials (hivecommons/hive#18015). +// +// A project.reference_repos entry that opted in to writes needs a credential +// GitHub actually honours for THAT repo. The hive's own installation token +// cannot be it: an App installation is per-account, and a reference repo is by +// definition outside the hive's org. Two sources are supported, in the order +// an operator reaches for them: +// +// - the same App installed on the target org — the hive discovers that +// installation and mints a token on it, scoped to the one listed repo and +// to exactly the permissions the opt-in needs; +// - write_token_file — an operator-supplied token (a maintainer PAT, a +// fine-grained token) read from disk on each use so a rotation is picked up +// without a restart. +// +// Boot refuses a write-mode entry that has neither (see +// VerifyReferenceRepoWriteCredential) so the failure is a config error with +// the repo named, not a 401 a day later. + +// referenceTokenRefreshMargin is how long before expiry a minted reference +// token is replaced. +const referenceTokenRefreshMargin = 5 * time.Minute + +// ErrReferenceRepoNoCredential is the sentinel for a write-mode entry with no +// usable credential source. +var ErrReferenceRepoNoCredential = errors.New("reference repo write-mode has no credential") + +// referenceRepoPermissions are the installation permissions an opt-in needs: +// metadata always, issues:write for issues, contents:write + pull_requests:write +// for prs (a PR needs a branch to be pushed first). Nothing wider — the proxy +// narrows further to the exact write paths. +func referenceRepoPermissions(rr config.ReferenceRepo) *gh.InstallationPermissions { + perms := &gh.InstallationPermissions{Metadata: gh.Ptr("read"), Contents: gh.Ptr("read")} + if rr.WritesIssues() { + perms.Issues = gh.Ptr("write") + } + if rr.WritesPRs() { + perms.Contents = gh.Ptr("write") + perms.PullRequests = gh.Ptr("write") + } + return perms +} + +// ReferenceRepoTokenSource returns a token function for the proxy: the +// write_token_file when set, else a cached installation token minted on the +// target org. auth may be replaced on key rotation, so it is taken through a +// getter rather than captured. +func ReferenceRepoTokenSource(rr config.ReferenceRepo, auth func() *AppAuth, logger *slog.Logger) func() (string, bool) { + if rr.ReadOnly() { + return nil + } + if rr.WriteTokenFile != "" { + return func() (string, bool) { + tok, err := readTokenFile(rr.WriteTokenFile) + if err != nil { + logger.Warn("reference repo write token unreadable", "repo", rr.Repo, "file", rr.WriteTokenFile, "error", err) + return "", false + } + return tok, true + } + } + var ( + mu sync.Mutex + token string + expiry time.Time + ) + return func() (string, bool) { + mu.Lock() + defer mu.Unlock() + if token != "" && time.Now().Add(referenceTokenRefreshMargin).Before(expiry) { + return token, true + } + a := auth() + if a == nil { + logger.Warn("reference repo write token: no App auth available", "repo", rr.Repo) + return "", false + } + ctx, cancel := context.WithTimeout(context.Background(), mintClientTimeout) + defer cancel() + tok, exp, err := a.MintReferenceRepoToken(ctx, rr) + if err != nil { + logger.Warn("reference repo write token mint failed", "repo", rr.Repo, "error", err) + return "", false + } + token, expiry = tok, exp + return token, true + } +} + +// MintReferenceRepoToken mints an installation token on the TARGET org's +// installation of this App, scoped to rr's single repository and to the +// permissions its write opt-in needs. +func (a *AppAuth) MintReferenceRepoToken(ctx context.Context, rr config.ReferenceRepo) (string, time.Time, error) { + if a == nil { + return "", time.Time{}, ErrNoGitHubClient + } + installationID, err := a.DiscoverInstallationID(ctx, rr.Owner()) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: %s: %v", ErrReferenceRepoNoCredential, rr.Repo, err) + } + opts := &gh.InstallationTokenOptions{ + Repositories: []string{rr.Name()}, + Permissions: referenceRepoPermissions(rr), + } + return a.mintInstallationTokenWithOptions(ctx, installationID, opts) +} + +func readTokenFile(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + tok := strings.TrimSpace(string(b)) + if tok == "" { + return "", fmt.Errorf("%s is empty", path) + } + return tok, nil +} + +// VerifyReferenceRepoWriteCredential is the boot-time check for one write-mode +// entry: the write_token_file must be readable and non-empty, or the App must +// have an installation on the target org. It returns the login the credential +// acts as — the App bot login, or the token's user — for the proxy's own-item +// check. discover and whoAmI are seams: production passes the App's installation +// discovery and a GET /user for file tokens. +func VerifyReferenceRepoWriteCredential(ctx context.Context, rr config.ReferenceRepo, botLogin string, discover func(ctx context.Context, org string) (int64, error), whoAmI func(ctx context.Context, token string) (string, error)) (string, error) { + if rr.ReadOnly() { + return "", nil + } + if rr.WriteTokenFile != "" { + tok, err := readTokenFile(rr.WriteTokenFile) + if err != nil { + return "", fmt.Errorf("%w: project.reference_repos %s: write_token_file %s: %v", ErrReferenceRepoNoCredential, rr.Repo, rr.WriteTokenFile, err) + } + if whoAmI == nil { + return "", fmt.Errorf("%w: project.reference_repos %s: cannot resolve the write_token_file identity", ErrReferenceRepoNoCredential, rr.Repo) + } + login, err := whoAmI(ctx, tok) + if err != nil || login == "" { + return "", fmt.Errorf("%w: project.reference_repos %s: write_token_file %s is not a usable GitHub token (%v)", ErrReferenceRepoNoCredential, rr.Repo, rr.WriteTokenFile, err) + } + return login, nil + } + if discover == nil { + return "", fmt.Errorf("%w: project.reference_repos %s: write: %v needs the GitHub App installed on %s (or write_token_file), and this hive has no App auth", ErrReferenceRepoNoCredential, rr.Repo, rr.Write, rr.Owner()) + } + if _, err := discover(ctx, rr.Owner()); err != nil { + return "", fmt.Errorf("%w: project.reference_repos %s: write: %v needs the GitHub App installed on %s (or write_token_file): %v", ErrReferenceRepoNoCredential, rr.Repo, rr.Write, rr.Owner(), err) + } + if botLogin == "" { + return "", fmt.Errorf("%w: project.reference_repos %s: App bot login unknown, cannot attribute the hive's own items", ErrReferenceRepoNoCredential, rr.Repo) + } + return botLogin, nil +} + +// TokenUserLogin resolves the login a user token acts as (GET /user). +func TokenUserLogin(ctx context.Context, token, apiURL string) (string, error) { + user, _, err := newTokenClient(token, apiURL).Users.Get(ctx, "") + if err != nil { + return "", err + } + return user.GetLogin(), nil +} + +// BotLogin returns the App bot login recorded by SetBotLogin ("" when unknown). +func (a *AppAuth) BotLogin() string { + if a == nil { + return "" + } + a.mu.Lock() + defer a.mu.Unlock() + return a.botLogin +} diff --git a/src/pkg/github/reference_repo_token_test.go b/src/pkg/github/reference_repo_token_test.go new file mode 100644 index 0000000000..0024f336aa --- /dev/null +++ b/src/pkg/github/reference_repo_token_test.go @@ -0,0 +1,120 @@ +package github + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestVerifyReferenceRepoWriteCredential(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + tokFile := filepath.Join(dir, "tok") + if err := os.WriteFile(tokFile, []byte("ghp_x\n"), 0o600); err != nil { + t.Fatal(err) + } + emptyFile := filepath.Join(dir, "empty") + if err := os.WriteFile(emptyFile, []byte(" \n"), 0o600); err != nil { + t.Fatal(err) + } + installed := func(_ context.Context, org string) (int64, error) { + if org == "hivecommons" { + return 42, nil + } + return 0, ErrNoInstallationForOrg + } + whoAmI := func(_ context.Context, token string) (string, error) { + if token == "ghp_x" { + return "nate", nil + } + return "", errors.New("bad credentials") + } + + cases := []struct { + name string + rr config.ReferenceRepo + botLogin string + discover func(context.Context, string) (int64, error) + wantLogin string + wantErr bool + }{ + {name: "read-only needs nothing", rr: config.ReferenceRepo{Repo: "hivecommons/docs"}, wantLogin: ""}, + {name: "app installed on target org", rr: config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"issues"}}, botLogin: "hive[bot]", discover: installed, wantLogin: "hive[bot]"}, + {name: "app not installed, no token file", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues"}}, botLogin: "hive[bot]", discover: installed, wantErr: true}, + {name: "no app auth at all", rr: config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"prs"}}, wantErr: true}, + {name: "token file wins over installation", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues", "prs"}, WriteTokenFile: tokFile}, discover: installed, wantLogin: "nate"}, + {name: "empty token file", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues"}, WriteTokenFile: emptyFile}, wantErr: true}, + {name: "missing token file", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues"}, WriteTokenFile: filepath.Join(dir, "nope")}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + login, err := VerifyReferenceRepoWriteCredential(ctx, tc.rr, tc.botLogin, tc.discover, whoAmI) + if tc.wantErr { + if err == nil { + t.Fatalf("want error, got login %q", login) + } + if !errors.Is(err, ErrReferenceRepoNoCredential) { + t.Errorf("error should wrap ErrReferenceRepoNoCredential: %v", err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if login != tc.wantLogin { + t.Errorf("login = %q, want %q", login, tc.wantLogin) + } + }) + } +} + +func TestReferenceRepoTokenSource_File(t *testing.T) { + tokFile := filepath.Join(t.TempDir(), "tok") + if err := os.WriteFile(tokFile, []byte("first\n"), 0o600); err != nil { + t.Fatal(err) + } + rr := config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"issues"}, WriteTokenFile: tokFile} + src := ReferenceRepoTokenSource(rr, func() *AppAuth { return nil }, testLogger()) + if tok, ok := src(); !ok || tok != "first" { + t.Fatalf("got %q,%v want first,true", tok, ok) + } + // Rotation is picked up without a restart. + if err := os.WriteFile(tokFile, []byte("second"), 0o600); err != nil { + t.Fatal(err) + } + if tok, ok := src(); !ok || tok != "second" { + t.Fatalf("got %q,%v want second,true", tok, ok) + } + if err := os.Remove(tokFile); err != nil { + t.Fatal(err) + } + if _, ok := src(); ok { + t.Fatal("removed token file should yield ok=false") + } + if ReferenceRepoTokenSource(config.ReferenceRepo{Repo: "a/b"}, nil, testLogger()) != nil { + t.Fatal("read-only entry must have no token source") + } +} + +func TestReferenceRepoTokenSource_NoAuth(t *testing.T) { + rr := config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"prs"}} + src := ReferenceRepoTokenSource(rr, func() *AppAuth { return nil }, testLogger()) + if _, ok := src(); ok { + t.Fatal("no App auth must yield ok=false, not a credential") + } +} + +func TestReferenceRepoPermissions(t *testing.T) { + p := referenceRepoPermissions(config.ReferenceRepo{Repo: "a/b", Write: []string{"issues"}}) + if p.GetIssues() != "write" || p.GetContents() != "read" || p.GetPullRequests() != "" { + t.Errorf("issues-only perms wrong: issues=%q contents=%q prs=%q", p.GetIssues(), p.GetContents(), p.GetPullRequests()) + } + p = referenceRepoPermissions(config.ReferenceRepo{Repo: "a/b", Write: []string{"prs"}}) + if p.GetIssues() != "" || p.GetContents() != "write" || p.GetPullRequests() != "write" { + t.Errorf("prs perms wrong: issues=%q contents=%q prs=%q", p.GetIssues(), p.GetContents(), p.GetPullRequests()) + } +} diff --git a/src/pkg/scheduler/agent_repo_scope_test.go b/src/pkg/scheduler/agent_repo_scope_test.go index a767d9a60b..40681457a0 100644 --- a/src/pkg/scheduler/agent_repo_scope_test.go +++ b/src/pkg/scheduler/agent_repo_scope_test.go @@ -243,3 +243,30 @@ func TestFilterIssuesForRepos(t *testing.T) { t.Errorf("filterIssuesForRepos = %v, want the b issue", got) } } + +// project.reference_repos (hivecommons/hive#18015): the kick names each +// reference repo with its tier and tells the agent an item whose deliverables +// mention one is in scope, so the scanner stops classifying it out-of-scope. +func TestBuildReposSectionFor_ReferenceRepos(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + section := s.buildReposSectionFor("scanner") + for _, want := range []string{ + "REFERENCE REPOS", + "https://github.com/hivecommons/docs [reference · read-only]", + "https://github.com/hivecommons/hive [reference · issues+prs]", + "IN SCOPE, not a scope violation", + } { + if !strings.Contains(section, want) { + t.Errorf("%q missing:\n%s", want, section) + } + } + // A hive without the key sees no trace of it. + s.cfg.Project.ReferenceRepos = nil + if strings.Contains(s.buildReposSectionFor("scanner"), "REFERENCE REPOS") { + t.Error("reference block emitted with no reference repos configured") + } +} diff --git a/src/pkg/scheduler/kickmessage.go b/src/pkg/scheduler/kickmessage.go index bf3f40a8d7..e1ee2ed765 100644 --- a/src/pkg/scheduler/kickmessage.go +++ b/src/pkg/scheduler/kickmessage.go @@ -846,6 +846,18 @@ func (s *Scheduler) buildReposSectionFor(agentName string) string { b.WriteString(fmt.Sprintf("⏸️ PAUSED BY THE OPERATOR (watched, but OUT OF SCOPE this session): %s\n", strings.Join(pausedFull, ", "))) b.WriteString(" The hive is deliberately quiet on those repos — a release freeze, an incident, a repo declared but not yet onboarded. Writes to them are refused deterministically by the proxy and by the hive-open-pr/hive-merge relays, so retrying cannot succeed. This is an operator decision, NOT an outage and NOT scope loss: do not work them, do not route around it, and do not file an issue about it.\n") } + // project.reference_repos (hivecommons/hive#18015): repos outside the + // project an agent may STUDY — e.g. a docs-assessment hive reading the + // repos it assesses. Without this block the scanner reads any item whose + // deliverables name them as out-of-scope/critical, and agents either + // refuse the work or try to write to them. + if refs := s.cfg.Project.ReferenceRepos; len(refs) > 0 { + b.WriteString("📚 REFERENCE REPOS (outside this project — study them, do not work them):\n") + for _, rr := range refs { + b.WriteString(fmt.Sprintf(" %s/%s [reference · %s]\n", strings.TrimRight(host, "/"), rr.Repo, rr.WriteLabel())) + } + b.WriteString(" You MAY clone them (public, unauthenticated), read their files, and GET their issues/PRs/API. An authorized-repo item whose deliverables name a reference repo is IN SCOPE, not a scope violation: do NOT classify it out-of-scope, critical, or a security concern. Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo (the assessment document, the report), never as issues or PRs on the reference repo itself — unless its tier above says issues/prs, in which case you may ONLY open an issue, open a PR from your own branch, or comment on an issue/PR you opened; the proxy refuses merges, default-branch pushes, labels, assignments, reviews and closing others' items deterministically, so retrying cannot succeed.\n") + } b.WriteString("⛔ NEVER access, search, list, file issues in, or open PRs on repos not listed above.\n") b.WriteString(fmt.Sprintf("⛔ Every repo above is on %s. This hive is single-host — never touch a repo on a different GitHub host.\n", host)) // SCOPE vs PROVISIONING (#4464). This list is what `include_repos: true` From bf6c10443b72472cae11148a6b377edd82cab9b4 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 16:47:57 -0700 Subject: [PATCH 128/223] =?UTF-8?q?=E2=9C=A8=20Owner-facing=20GitHub=20web?= =?UTF-8?q?hook=20setup:=20Settings=20section,=20generated=20secret,=20cre?= =?UTF-8?q?ate-hook,=20owner=20guide=20(#18024)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Owner-facing GitHub webhook setup: Settings section, generated secret, create-hook, owner guide Settings → Forge App gains a GitHub webhooks section with the payload URL, events, live receiver status, Generate secret (file-backed, shown once) and Create webhook for me when the App installation has hooks write permission. Governor card gets a webhooks tile deep-linking to it. Owner guide added to docs/webhooks.md. Hosted spokes keep receiving hub relays with no action. Fixes #18017 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * Document webhook setup routes in openapi.json; move section styling to tokens Fixes TestOpenAPISpecCoversEveryRegisteredRoute, TestStyleRatchet and TestDashboardDatesUseSharedTimezoneHelpers from the first CI run. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-webhook-owner-setup.md | 1 + dashboard/openapi.json | 258 +++++++++++++ src/docs/api-reference/agents-and-controls.md | 26 +- src/docs/api-reference/beads.md | 10 +- src/docs/api-reference/configuration.md | 29 +- src/docs/api-reference/dashboard-misc.md | 24 +- src/docs/api-reference/inception.md | 28 +- src/docs/api-reference/knowledge.md | 90 ++--- src/docs/api-reference/nous.md | 50 +-- src/docs/api-reference/packs-and-acmm.md | 36 +- src/docs/dashboard-source-layout.md | 4 +- src/docs/env-vars.md | 2 +- .../operator-reference/runtime-and-quota.md | 17 +- src/docs/webhooks.md | 77 ++++ src/pkg/dashboard/api.go | 3 + src/pkg/dashboard/api_github_webhook.go | 8 +- src/pkg/dashboard/api_github_webhook_setup.go | 339 ++++++++++++++++++ .../api_github_webhook_setup_test.go | 183 ++++++++++ .../static-src/css/config-webhooks.css | 27 ++ .../dashboard/static-src/js/config-dialog.js | 1 + .../dashboard/static-src/js/config-tabs.js | 5 +- .../static-src/js/config-webhooks.js | 217 +++++++++++ .../dashboard/static-src/js/governor-view.js | 25 ++ src/pkg/dashboard/static-src/manifest.txt | 2 + src/pkg/dashboard/static/index.html | 275 +++++++++++++- src/pkg/github/app.go | 2 + src/pkg/github/webhook_relay_key.go | 5 +- src/pkg/github/webhook_secret.go | 133 +++++++ src/pkg/github/webhook_secret_test.go | 105 ++++++ src/pkg/github/webhook_setup.go | 146 ++++++++ 30 files changed, 1965 insertions(+), 163 deletions(-) create mode 100644 changelog.d/added-webhook-owner-setup.md create mode 100644 src/pkg/dashboard/api_github_webhook_setup.go create mode 100644 src/pkg/dashboard/api_github_webhook_setup_test.go create mode 100644 src/pkg/dashboard/static-src/css/config-webhooks.css create mode 100644 src/pkg/dashboard/static-src/js/config-webhooks.js create mode 100644 src/pkg/github/webhook_secret.go create mode 100644 src/pkg/github/webhook_secret_test.go create mode 100644 src/pkg/github/webhook_setup.go diff --git a/changelog.d/added-webhook-owner-setup.md b/changelog.d/added-webhook-owner-setup.md new file mode 100644 index 0000000000..11c7755899 --- /dev/null +++ b/changelog.d/added-webhook-owner-setup.md @@ -0,0 +1 @@ +- Owner-facing GitHub webhook setup: ⚙ Settings → Forge App gains a "GitHub webhooks" section showing the payload URL, events and live receiver status, with **Generate secret** (stored in `/data/github-webhook-secret.key`, shown once) and **Create webhook for me** when the App installation has hooks write permission; a webhooks tile on the Governor card deep-links to it; new owner guide in `docs/webhooks.md` (#18017). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 651888249f..6167fff8a0 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -25525,6 +25525,264 @@ } } }, + "/api/config/github/webhooks": { + "get": { + "tags": [ + "System" + ], + "summary": "Describe GitHub webhook setup for this hive (owner)", + "description": "Owner-only. Everything the Settings → Forge App → GitHub webhooks section shows: the payload URL GitHub must call, the events to subscribe, whether a secret is configured and where it comes from (never the secret itself), whether this hive receives hub-signed relays (hosted), the live receiver health and whether the App installation's hooks permission allows creating the webhook from here. See docs/webhooks.md#setting-up-webhooks-owner-guide.", + "responses": { + "200": { + "description": "Webhook setup", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "payload_url": { + "type": "string" + }, + "payload_url_source": { + "type": "string", + "enum": [ + "config", + "hub", + "request", + "none" + ] + }, + "path": { + "type": "string" + }, + "content_type": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "optional_events": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "verification": { + "type": "string" + }, + "hosted": { + "type": "boolean" + }, + "secret_configured": { + "type": "boolean" + }, + "secret_source": { + "type": "string", + "enum": [ + "env", + "file", + "" + ] + }, + "secret_path": { + "type": "string" + }, + "can_generate_secret": { + "type": "boolean" + }, + "health": { + "type": "object", + "properties": { + "healthy": { + "type": "boolean" + }, + "last_event_at": { + "type": "string" + }, + "events_1h": { + "type": "integer" + }, + "invalidations_1h": { + "type": "integer" + }, + "targeted_refreshes_1h": { + "type": "integer" + }, + "tracker_invalidations_1h": { + "type": "integer" + }, + "last_ci_event_at": { + "type": "string" + }, + "ci_events_1h": { + "type": "integer" + }, + "ci_events_applied_1h": { + "type": "integer" + }, + "ci_refetches_1h": { + "type": "integer" + }, + "verification": { + "type": "string" + } + } + }, + "org": { + "type": "string" + }, + "repos": { + "type": "array", + "items": { + "type": "string" + } + }, + "app_org_hooks": { + "type": "boolean" + }, + "app_repo_hooks": { + "type": "boolean" + }, + "app_probe_error": { + "type": "string" + }, + "create_available": { + "type": "boolean" + }, + "docs_url": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not an owner" + }, + "503": { + "description": "Config not loaded" + } + } + } + }, + "/api/config/github/webhooks/secret": { + "post": { + "tags": [ + "System" + ], + "summary": "Generate this hive's GitHub webhook secret (owner)", + "description": "Owner-only. Writes a new random secret to the hive's secret file (mode 0600) and returns it exactly once; later GETs only report that a file secret is configured. Refused with 409 while GITHUB_WEBHOOK_SECRET is set in the environment, which always wins. Audited.", + "responses": { + "200": { + "description": "Secret, shown once", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "secret": { + "type": "string" + }, + "secret_path": { + "type": "string" + }, + "verification": { + "type": "string" + }, + "note": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not an owner" + }, + "409": { + "description": "Secret is supplied by the environment" + }, + "500": { + "description": "Could not write the secret file" + } + } + } + }, + "/api/config/github/webhooks/create": { + "post": { + "tags": [ + "System" + ], + "summary": "Create the org or repo webhook through the GitHub App (owner)", + "description": "Owner-only. Uses the App installation's organization_hooks or repository_hooks write permission to create a webhook on GitHub pointing at this hive's payload URL with the configured secret and event list. Requires a configured secret, a public dashboard URL and the matching permission; the repository must be one of project.repos. Audited.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "scope": { + "type": "string", + "enum": [ + "organization", + "repository" + ] + }, + "repo": { + "type": "string", + "description": "Repository name (bare or org/name); required for scope=repository." + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Webhook created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hook_id": { + "type": "integer" + }, + "scope": { + "type": "string" + }, + "target": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "Invalid scope or repository" + }, + "403": { + "description": "Not an owner, or the App lacks the hooks permission" + }, + "409": { + "description": "No secret / no public URL / webhook already exists" + }, + "502": { + "description": "GitHub API error" + } + } + } + }, "/api/config/github": { "put": { "tags": [ diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index 7f42dc9717..01bcaaa5f4 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -23,20 +23,20 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:175` | | `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:176` | | `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:239` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:515` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:518` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:519` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:523` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:518` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:521` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:522` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:526` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:527` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:528` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:530` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:533` | -| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:534` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:320` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:321` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:322` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:323` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:530` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:531` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:533` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:536` | +| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:537` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:323` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:324` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:325` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:326` | | `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:126` | | `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:130` | diff --git a/src/docs/api-reference/beads.md b/src/docs/api-reference/beads.md index b67e37d79d..443e2a5459 100644 --- a/src/docs/api-reference/beads.md +++ b/src/docs/api-reference/beads.md @@ -5,8 +5,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:499` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:500` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:501` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:502` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:503` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:502` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:503` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:504` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:505` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:506` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 4941f8f958..02117e6006 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -75,17 +75,20 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:314` | | `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:315` | | `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:318` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:373` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:374` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:375` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:365` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:366` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:367` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:368` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:369` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:370` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:371` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:541` | +| `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:87` | +| `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:88` | +| `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:89` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:376` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:377` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:378` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:368` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:369` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:370` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:371` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:372` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:373` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:374` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:544` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:220` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:221` | @@ -94,8 +97,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:230` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:234` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:537` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:538` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:540` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:541` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:237` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:240` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:246` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index dc4eacc334..0b4e24ec7e 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -25,7 +25,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](../audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:63` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:510` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:513` | | `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:66` | | `POST` | `/api/upgrade/announce` | Owner role | Announce a pod roll before it happens ([#15788](https://github.com/hivecommons/hive/issues/15788)): body `{"target": "<sha|tag>", "source": "hub"|"operator", "reason": "..."}` (`version` is an alias of `target`; all fields optional). Marks `/api/status`/`/api/version` `upgradeProgress` as `state: pulling, phase: announced` with the target and publishes an SSE `upgrading` event so every open dashboard shows the upgrade animation; a SIGTERM afterwards (or with no announce at all) publishes `phase: restarting` from the same record. The hub sends the same announce through the heartbeat response (`upgrade_announce`) for hub-managed hives; an announce with no kill expires after 15 minutes. Returns `{ok, upgradeProgress}` | `pkg/dashboard/api.go:69` | | `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:67` | @@ -40,18 +40,18 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:264` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:462` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:465` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:466` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:468` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:469` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:465` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:468` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:469` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:471` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:472` | | `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:215` | | `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:216` | | `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:217` | | `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:306` | | `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {<repo>: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:307` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:548` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:551` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:118` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:123` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:137` | @@ -62,11 +62,11 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, `claimant: {kind: hive-agent|clanker|person, login, agent, hive, since, source: claim-ledger|claim-comment|assignee|kick-history|author}` naming who holds a claimed / in-progress issue or a PR under review or rework (absent when nobody is named; mirrored onto `linked_prs[]`), (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `decision_needed` on a row parked on a person (085 / 045 / 015, the needs-human pill; mirrored onto the issue's `linked_prs[]` entry): `{question, asked_by, since, blocker, blocker_reason, options: [{action, label, enabled, reason_if_disabled}]}` — the reviewer's own decision question when the verdict record has one, the blocker in plain words, and the applicable actions under the row context menu's action ids (`decide:delegate` first), (PRs) `lineage` on the hive's copy of an adopted PR — `{kind: adopted|superseded|rebased-by-hive, from_number, from_url, from_author, from_author_kind, to_number, to_url, at, reason}`, derived from the stage store's adoption record or the `hive/adopt-<n>` head branch, mirrored onto the issue's `linked_prs[]` entry; absent on every other row — and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close; a PR in a conflict-aware merge train carries `merge_train` `{id, state, front, position, size, attempt, max_attempts, overlap_files, ahead, length, head_number, head_state, head_reason, head_since, eta_s, eta_at, eta_basis, late, stalled, stalled_s}` — `head_state` one of `needs-human` / `ci` / `soaking` / `ready` / `merging` / `fixing` / `unknown` with `head_reason` in plain words; `eta_s` / `eta_at` the estimated wait for this member's own merge attempt (own remaining soak + each PR ahead's remaining soak or one sweep + the next sweep tick; absent while the head waits on a person or CI), `eta_at` anchored to the member's first estimate at its position so a countdown survives polls, `eta_basis` its short reading ("2 ahead · head soaking 12m · next sweep in 40s"), `late` once now has overrun the first estimate twice over, `stalled` / `stalled_s` once the head has sat longer than `governor.merge_train_stall_warn_s` ([#15395](https://github.com/hivecommons/hive/issues/15395)); a PR the queue holds because another open PR declares the same `Fixes #N` carries no `merge_train` and a `merge_gate` of `{state: "duplicate-of:<winner>", duplicate_of, duplicate_issue}` plus the `needs-decision` modifier (the queue keeps one PR per issue: green first, then the earlier-opened); a PR GitHub reports conflicting on its own, or one under repair (stage/070, 071, 072, 074, 085, or a red required check), carries no `merge_train` and is not counted in any member's `size` / `length` — a repaired member the view dropped carries `left_train: {why: fixing|needs-author|returned-to-contributor|adopted-by-hive|needs-decision|checks-failing, note}` instead (mirrored onto the issue's `linked_prs[]` entry), and a train left with one member reports no `merge_train` at all; a PR short of the merge queue (below stage/080) the planner ordered rides no train either — it carries `overlap: {behind, with, files, note}` ("overlaps #15637 (sidebar.html, index.html) — will queue behind it") and no `merge_train`, and counts in no member's `size` — the state slugs (`front`, `queued-behind`) are payload values only, the SPA renders ordinal places ("Queued for merge · 1st of 2", with the train's tag when more than one train is on the page). Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=<remaining>`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:191` | | `GET` | `/api/matrix/lines` | Dashboard auth/session | Matrix Lines (deprecated name, see [Published-checkpoint regions](#published-checkpoint-regions)): serve bounded backlog of pushed lines behind dashboard.matrix with repo allowlist filtering. `max` counts only lines the repo allowlist admits, so a page is never emptied by filtered-out lines. A client paging with `max` should use the last returned line's `seq` as its next `since`, not the top-level `seq`, which is the source's latest sequence number and would skip lines still past the page. `more` is true only while another line the caller may see remains past the page; stop paging when it is false. | `pkg/dashboard/api.go:192` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:193` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:378` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:441` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:442` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:471` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:505` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:381` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:444` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:445` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:474` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:508` | | `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:350` | | `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:351` | | `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:352` | diff --git a/src/docs/api-reference/inception.md b/src/docs/api-reference/inception.md index 0260a12e83..6da92d2d96 100644 --- a/src/docs/api-reference/inception.md +++ b/src/docs/api-reference/inception.md @@ -5,17 +5,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:444` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:445` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:446` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:447` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:448` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:449` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:450` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:451` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:452` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:453` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:454` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:455` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:456` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:457` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:447` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:448` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:449` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:450` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:451` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:452` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:453` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:454` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:455` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:456` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:457` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:458` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:459` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:460` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index e406cdb1a3..a20c2621ba 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -5,38 +5,38 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:380` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:381` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:390` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:382` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:383` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:384` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:385` | -| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:387` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:391` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:392` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:393` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:394` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:395` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:396` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:397` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:398` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:404` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:405` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:406` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:407` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:408` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:409` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:410` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:411` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:412` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:413` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:414` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:415` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:416` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:417` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:418` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:419` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:383` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:384` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:393` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:385` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:386` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:387` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:388` | +| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:390` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:394` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:395` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:396` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:397` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:398` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:399` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:400` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:401` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:407` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:408` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:409` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:410` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:411` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:412` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:413` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:414` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:415` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:416` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:417` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:418` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:419` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:420` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:421` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:422` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -46,19 +46,19 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](../settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:429` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:430` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:431` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:432` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:433` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:434` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:435` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:436` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:437` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:438` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:439` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:402` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:403` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:432` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:433` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:434` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:435` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:436` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:437` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:438` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:439` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:440` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:441` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:442` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:405` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:406` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | diff --git a/src/docs/api-reference/nous.md b/src/docs/api-reference/nous.md index 6dce3749dd..6400d1c981 100644 --- a/src/docs/api-reference/nous.md +++ b/src/docs/api-reference/nous.md @@ -5,28 +5,28 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:473` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:474` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:475` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:476` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:477` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:478` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:479` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:480` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:481` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:482` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:483` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:484` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:485` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:486` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:487` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:488` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:489` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:490` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:491` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:492` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:493` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:494` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:495` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:496` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:497` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:476` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:477` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:478` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:479` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:480` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:481` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:482` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:483` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:484` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:485` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:486` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:487` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:488` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:489` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:490` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:491` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:492` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:493` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:494` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:495` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:496` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:497` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:498` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:499` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:500` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 7dbfa576e4..b9bfed2203 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -5,29 +5,29 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:325` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:326` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:327` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:328` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:329` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:330` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](../repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:342`, `pkg/dashboard/api_repo_hold.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:345`, `pkg/dashboard/api_repo_hold.go` | | `GET` | `/api/repos/{owner}/{repo}/items/{number}` | Dashboard session | Per-item detail for the activity monitor's decision badge: `{repo, number, stage, stage_slug, operator_actions[], operator_action_count, last_operator_action}`. `operator_actions[]` is the server-side ledger of every row action submitted from the dashboard (hold/release, decision, decide-delegated, send-to-human, accept-triage — recorded by `recordOperatorAction` around those routes) as `{action, actor, at, note, target, outcome: pending|applied|no-op|failed, detail, head_sha, stage}`; `last_operator_action` adds the derived `state` (`waiting|addressed|stuck|pending|failed`), `stuck_after_seconds` and `followed[]` (the stage hops after the action). The same `last_operator_action` / `operator_action_count` ride every stage-tracker row and linked PR. Persisted in `/data/operator-actions.json`; `HIVE_OPERATOR_ACTION_STUCK_AFTER` (default `30m`) sets the stuck window. Rows also carry `rebase_merge: {state: approved|rebasing|ci-pending|merged|failed, actor, base, text, fail_reason, conflict_files[], armed_at}` while a *Rebase, then merge as maintainer* follow-through is live or just settled (persisted in `/data/rebase-merge-followups.json`; `fail_reason: foreign-head` when the head moved after the approval by someone other than the hive, so nothing merged), and `rebase: {attempt, max, exhausted, head, text}` on a conflicting PR the fixer is rebasing under the two-attempt loop guard (`/data/rebase-attempts.json`). | `pkg/dashboard/api.go`, `pkg/dashboard/operator_actions.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:346`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:347`, `pkg/dashboard/api_repo_item_decision.go` | -| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_maintainer_notes.go` | -| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:352`, `pkg/dashboard/api_maintainer_notes.go` | -| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:353`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:349`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:350`, `pkg/dashboard/api_repo_item_decision.go` | +| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:354`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:355`, `pkg/dashboard/api_maintainer_notes.go` | +| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_maintainer_notes.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `<!-- hive:delegated-decision … -->` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park, ladder?: {decision, reason, ruled_out[]}}, notes, error}`; `ladder` is set on contributor-PR decisions and names the rung that decided — `already-fixed` → `mergeable-as-is` → `rebase-then-merge` → `return-to-contributor` → `contributor-offline` — and why each rung above it did not apply); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:557`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:362` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:364`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:359`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:360` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:361` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:560`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:363`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:364` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:365` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_hive_advice.go` | diff --git a/src/docs/dashboard-source-layout.md b/src/docs/dashboard-source-layout.md index e75eca06f9..2e907c305a 100644 --- a/src/docs/dashboard-source-layout.md +++ b/src/docs/dashboard-source-layout.md @@ -198,7 +198,7 @@ CSS (`css/`): `base` (tokens, reset, shared primitives), `sidebar`, `topbar`, `agent-detail`, `agent-cards`, `auth`, `agent-state`, `tracker` (Hive activity monitor), `tracker-native-queue`, `agent-fields`, `governor`, `advisory-lifecycle`, `evidence`, `throughput`, `repos`, `label-filter`, `overview`, `sparklines-indicators`, -`governor-matrix`, `cost`, `alerts-toasts`, `feedback-faq`, `config-dialog`, +`governor-matrix`, `cost`, `alerts-toasts`, `feedback-faq`, `config-dialog`, `config-webhooks`, `chat`, `nous`, `knowledge`, `sections-collapse`, `design-system`, `responsive`, `recipes-utilities`, `welcome`, `review-campaigns`, `variables`. @@ -224,7 +224,7 @@ JS (`js/`): `boot`, `core`, `feedback`, `sections`, `layout`, `agent-detail`, `review-queue`, `contributors`, `refresh`, `inception`, `debug-logs`, `tooltips-dialogs`, `plan-agent-controls`, `acmm`, `version`, `sse-bootstrap`, `config-state`, `settings-search`, `config-dialog`, `config-tabs`, -`config-hub`, `config-agent`, `config-governor`, `config-watchdog-security`, +`config-webhooks`, `config-hub`, `config-agent`, `config-governor`, `config-watchdog-security`, `config-features`, `config-sentinel`, `compliance`, `config-helpers`, `linear-repos-save`, `chat`, `gh-auth`, `nps`, `welcome`, `shimmy-handlers`. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 23d80031c7..e9e8264b48 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -104,7 +104,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTE_SKIP_LABELS` | No | `blocked,tracking,epic,discussion,question,needs-decision,needs-triage` | Comma-separated, case-insensitive label patterns for issues that are not contributor work and must never be offered by the relay. Patterns use `path.Match`-style `*` globs; `blocked` is always unioned into the effective set even if omitted. Same setting as `hub.contribute_skip_labels`. | | `HIVE_FEDERATION_REGISTRY_PATH` | No | `/data/federation/registry.json` | Federation registry path override. | | `HIVE_WEBHOOK_SECRET` | No | none | HMAC secret for the spoke `/webhook` channel. | -| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a self-hosted spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret. A hosted spoke leaves it unset and verifies hub relays with its own per-hive relay key from the heartbeat; with neither, every delivery is rejected. | +| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub); `/data/github-webhook-secret.key` when present (spoke, written by Settings → Forge App → **Generate secret**) | GitHub App or org/repo webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a self-hosted spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret. When the variable is set it wins over the generated file and the dashboard's Generate button is disabled. See [Setting up webhooks](webhooks.md#setting-up-webhooks-owner-guide). A hosted spoke leaves it unset and verifies hub relays with its own per-hive relay key from the heartbeat; with neither, every delivery is rejected. | | `HIVE_DASHBOARD_URL` | No | none | Base URL the `hive tui` client targets (`pkg/tui/client`). A bad value surfaces as a request error on the first call, not at startup. On the hub it is also consulted (fourth) in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_CONVERGENCE_MODE` | No | `convergence.mode` in `hive.yaml`, else `shadow` | Process-level override of the convergence mode (`off`, `shadow`, `enforce`) so an operator can flip the mode without editing `hive.yaml`. An **unset** mode takes the default, `shadow`, which computes and records admission decisions without withholding any work. A **non-empty but unrecognised** value — a typo, or a mode this build does not know — still resolves to `off`. | | `HIVE_RELEASE_LINE_LAG_MAX` | No | `5` | Alarm threshold for the release-line drift surface ([#6960](https://github.com/hivecommons/hive/issues/6960)): how many commits the hosted edge line (`v5`) may sit behind the stable default branch (`v4`) before the dashboard status flags the line as drifting (`releaseLineLag.exceeded`). An unset, empty, non-numeric, or negative value falls back to the default — it can never silently disable the alarm. An unknown lag (tips unresolved or the compare failed) is reported as `unknown`, never a healthy zero. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index bd85fb4553..f51851ece5 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -257,7 +257,11 @@ GraphQL query fails, or GitHub returns `mergeable: UNKNOWN` for that PR. See [GitHub webhooks: cadence, patch and reconcile](../webhooks.md) for the cadence keys, the fallback to polling when deliveries degrade, and the -accuracy guarantees of the ForgeState contract. +accuracy guarantees of the ForgeState contract. Hive owners (not cluster +operators) should start at the +[owner guide](../webhooks.md#setting-up-webhooks-owner-guide): it covers the +⚙ Settings → **Forge App** → **GitHub webhooks** section, which shows the +payload URL, generates a per-hive secret and can create the org/repo hook. A spoke accepts GitHub App webhooks on the public `POST /api/webhook/github`, either delivered directly by GitHub or relayed by the hub (the hub forwards the @@ -270,10 +274,13 @@ Deliveries for repositories the hive does not manage are ignored. #### What the operator configures -1. **Secret (self-hosted only).** Set `GITHUB_WEBHOOK_SECRET` in the spoke's - environment (a Secret mount on Kubernetes; never in `hive.yaml`) to the - App's webhook secret. A hosted spoke needs nothing here: it never holds the - App secret and verifies relays with its hub relay key instead (item 3). +1. **Secret (self-hosted only).** Either press **Generate secret** in + ⚙ Settings → Forge App → GitHub webhooks (the hive writes + `/data/github-webhook-secret.key`, mode 0600, and shows the value once), or + set `GITHUB_WEBHOOK_SECRET` in the spoke's environment (a Secret mount on + Kubernetes; never in `hive.yaml`). The environment variable wins when both + exist. A hosted spoke needs nothing here: it never holds a GitHub secret and + verifies relays with its hub relay key instead (item 3). 2. **GitHub App webhook.** Point the App's webhook URL at the hub (`POST /api/github/webhook`) for hosted spokes, or at the spoke's own `/api/webhook/github` for a stand-alone hive (a repository or organization diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 53743ab215..37865c7593 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -13,6 +13,83 @@ The priority order throughout is **accuracy → latency → API thrift**. A delivery is a free hint; every outbound GitHub call is budgeted. The rules below exist so that the hive never trades the first for the third. +## Setting up webhooks (owner guide) + +Webhooks are optional. Without them a hive polls GitHub every +`governor.eval_interval_s` (default 300 s) and finds out about a review, a red +check or a new comment at the next tick. With them the hive is told at once: +the governor wakes early, CI-failure and review kicks land within seconds +instead of minutes, the base tick stretches to `governor.eval_interval_webhook_s` +(default 900 s), and the outbound API budget drops because each delivery costs +one confirming `GET` instead of a full re-list. Open ⚙ Settings → **Forge App** +→ **GitHub webhooks** on your hive, or click the **webhooks** tile on the +Governor card, to see which of the two cases below applies to you. + +Hive never shares its own webhook secret. Every hive verifies deliveries with +a key that belongs to it alone. + +### Hosted hive (hub-managed) + +Nothing to do. GitHub sends App deliveries to the hub, the hub verifies them +and relays each one to every hive that manages that repository, signed with +the relay key it minted for your hive when it was provisioned. The Settings +section reports **Receiver: hub relay** and the events/hour count; the +Governor tile reads `N/1h`. If it stays at **no events yet** for an hour of +activity, confirm the hivecommons GitHub App is installed on the organization +or repositories your hive tracks (Settings → Forge App → Installation) — the +hub can only relay what GitHub sends it. + +### Self-hosted hive + +Your hive needs an org- or repo-level webhook in GitHub that points at it and +uses a secret only you hold. + +1. **Generate the secret.** Settings → Forge App → GitHub webhooks → + **Generate secret**. The hive writes it to `/data/github-webhook-secret.key` + (0600) and shows it once with a copy button; it is never shown again + (regenerate to rotate). If you prefer to manage it yourself, set + `GITHUB_WEBHOOK_SECRET` in the hive's environment instead — the button then + reports that the environment variable is in charge. +2. **Create the webhook.** If the section offers **Create webhook for me**, + click it: the hive's GitHub App installation holds `organization_hooks` or + `repository_hooks` write permission and the hive creates the hook with the + right URL, secret and events. Otherwise do it by hand in GitHub: + organization (or repository) → Settings → Webhooks → **Add webhook**. + - Payload URL: the value shown in the section, i.e. + `https://<your-hive>/api/webhook/github` + - Content type: `application/json` + - Secret: the value from step 1 + - SSL verification: enabled + - Events: *Let me select individual events* → `check_run`, `check_suite`, + `issue_comment`, `issues`, `pull_request`, `pull_request_review`, + `pull_request_review_comment`, `push`, `status`, `workflow_run` + (optionally `workflow_job`, and `projects_v2_item` if your work source is + GitHub Projects). +3. **Verify.** GitHub sends a `ping` immediately; the section's **Receiver** + card should read **github-signed** with a fresh *last event*, and the + Governor tile moves from **set up →** to `N/1h` after the first real event. + +The payload URL comes from `dashboard.public_url`; if the section shows no URL +or the wrong one, set that key (or the hub-reported dashboard URL) to the +address GitHub can reach. + +### Troubleshooting + +| Symptom (GitHub → Settings → Webhooks → Recent deliveries) | Cause | Fix | +| --- | --- | --- | +| `401 invalid signature` | Secret in GitHub differs from the hive's | Paste the current secret into the GitHub hook (Edit → Secret), or regenerate in Settings and update GitHub. `GITHUB_WEBHOOK_SECRET` overrides the generated file — check which one the section says is active. | +| `503 webhooks disabled: no secret configured` | Hive has neither a generated/env secret nor a hub relay key | Step 1 above. | +| `404` | Wrong path | The path is `/api/webhook/github` on the spoke (the hub's App endpoint is `/api/github/webhook` — do not point an org hook there). | +| `200` but **no events yet** | Deliveries are for repos the hive does not manage | Check `project.repos`; deliveries for other repos are acknowledged and dropped. | +| Hosted hive, no events | App not installed on the repo / relay rejected | Install the App on the org or repo; the hub relays to the dashboard URL your hive reports, which must be reachable from the hub. | +| Deliveries time out | Hive not reachable from GitHub (private network, redirects) | GitHub (and the hub relay) follow no redirects and need a public HTTPS endpoint. | + +Hub relay vs direct: a hive registered with the hub whose repos carry the +hivecommons App receives relays as well as any direct hook you add; both are +accepted, duplicates are dropped by delivery id. Only one is needed. A +self-hosted hive with a private dashboard URL cannot receive relays and must +use a direct hook. + ## Polling cadence vs webhook patch vs reconcile Three writers feed one snapshot per repository. A fourth path writes PR diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index ceebd002c0..0808453692 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -316,6 +316,9 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // Read-only inventory for the Forge App tab: every App credential this // spoke holds (active config + per-app-id PVC keys), fingerprints only. s.mux.HandleFunc("GET /api/config/github/forge-apps", s.handleConfigGitHubForgeApps) + // Owner-facing webhook setup for the same tab: payload URL, events, + // secret generation and App-side hook creation (hivecommons/hive#18017). + s.registerGitHubWebhookSetupRoutes() s.mux.HandleFunc("GET /api/agents", s.handleAgentsList) s.mux.HandleFunc("POST /api/agents", s.handleAgentCreate) diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index 539de1084a..d0c7a166ba 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -8,7 +8,6 @@ import ( "io" "log/slog" "net/http" - "os" "strconv" "strings" "time" @@ -20,7 +19,8 @@ import ( const ( // githubWebhookPath is the PUBLIC path GitHub (or the hub relay) posts App // webhooks to. Not actually open: the handler fails closed without - // GITHUB_WEBHOOK_SECRET or a hub relay key, and verifies + // a webhook secret (GITHUB_WEBHOOK_SECRET or the dashboard-generated + // secret file, github.WebhookSecret) or a hub relay key, and verifies // X-Hub-Signature-256 or the hub relay signature over the raw body. githubWebhookPath = "/api/webhook/github" @@ -61,12 +61,12 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { jsonError(w, "failed to read body", http.StatusBadRequest) return } - secret := strings.TrimSpace(os.Getenv(githubWebhookSecretEnvVar)) + secret := github.WebhookSecret() // A hosted spoke holds no App webhook secret; it verifies the hub's relay // signature with this hive's own relay key instead (#17911). relayKey := github.WebhookRelayKey() if secret == "" && relayKey == "" { - const msg = "github webhook rejected: no webhook secret configured (set GITHUB_WEBHOOK_SECRET to enable signed webhooks)" + const msg = "github webhook rejected: no webhook secret configured (set GITHUB_WEBHOOK_SECRET or generate one under Settings → Forge App → GitHub webhooks)" warned := false s.webhookNoSecretOnce.Do(func() { warned = true diff --git a/src/pkg/dashboard/api_github_webhook_setup.go b/src/pkg/dashboard/api_github_webhook_setup.go new file mode 100644 index 0000000000..640157b176 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_setup.go @@ -0,0 +1,339 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +// Owner-facing GitHub webhook setup (hivecommons/hive#18017): the Settings → +// Forge App → "GitHub webhooks" section. Everything an owner needs to point +// a GitHub org/repo webhook at this hive with a secret of their own, the +// live receiver status, and — when the App installation may manage hooks — +// a one-click hook creation. The secret value leaves the hive exactly once, +// in the response to the generate call; every other response reports only +// whether and from where one is configured. + +const ( + auditWebhookSecretGenerated = "github_webhook_secret_generated" + auditWebhookCreated = "github_webhook_created" + + // webhookPermissionProbeTimeout bounds the installation lookup the GET + // makes to decide whether to offer "Create webhook for me". + webhookPermissionProbeTimeout = 8 * time.Second + webhookCreateTimeout = 20 * time.Second +) + +// webhookSetupResponse is GET /api/config/github/webhooks. +type webhookSetupResponse struct { + // PayloadURL is the public receiver URL to paste into GitHub, or "" when + // this hive has no known public origin (PayloadURLSource says why). + PayloadURL string `json:"payload_url"` + PayloadURLSource string `json:"payload_url_source"` + Path string `json:"path"` + ContentType string `json:"content_type"` + // Events to tick; Optional names events that matter only with a feature. + Events []string `json:"events"` + OptionalEvents map[string]string `json:"optional_events"` + + // Verification is /api/status.webhooks.verification: github-signed, + // relay (hub-signed) or none. Hosted reports whether a hub relay key is + // held, i.e. deliveries arrive through the hub without owner action. + Verification string `json:"verification"` + Hosted bool `json:"hosted"` + // SecretConfigured / SecretSource ("env", "file", "") describe the + // GitHub-signed path; CanGenerateSecret is false while the env var owns + // the secret. + SecretConfigured bool `json:"secret_configured"` + SecretSource string `json:"secret_source"` + SecretPath string `json:"secret_path,omitempty"` + CanGenerateSecret bool `json:"can_generate_secret"` + + Health ghpkg.WebhookHealth `json:"health"` + + // Org / Repos are the targets a hook would be created for. + Org string `json:"org"` + Repos []string `json:"repos"` + // App hook-management permissions, probed live; CreateAvailable is true + // when at least one is granted and a secret is configured. + AppOrgHooks bool `json:"app_org_hooks"` + AppRepoHooks bool `json:"app_repo_hooks"` + AppProbeError string `json:"app_probe_error,omitempty"` + CreateAvailable bool `json:"create_available"` + + DocsURL string `json:"docs_url"` +} + +const ( + webhookDocsURL = "https://github.com/hivecommons/hive/blob/v5/src/docs/webhooks.md#setting-up-webhooks-owner-guide" + // Where PayloadURL came from, for the card's explanation. + payloadURLSourceConfig = "dashboard.public_url" + payloadURLSourceHub = "hub dashboard URL" + payloadURLSourceRequest = "this request's host" + payloadURLSourceNone = "unknown — set dashboard.public_url" +) + +// webhookPermissionProbe is the installation lookup; tests swap it. +var webhookPermissionProbe = installationPermissionProbe + +func (s *Server) registerGitHubWebhookSetupRoutes() { + s.mux.HandleFunc("GET /api/config/github/webhooks", s.handleConfigGitHubWebhooks) + s.mux.HandleFunc("POST /api/config/github/webhooks/secret", s.handleConfigGitHubWebhooksGenerateSecret) + s.mux.HandleFunc("POST /api/config/github/webhooks/create", s.handleConfigGitHubWebhooksCreate) +} + +// webhookPayloadURL resolves this hive's public receiver URL: the configured +// dashboard.public_url, else the hub-assigned dashboard URL, else the origin +// the request arrived on (behind a proxy, X-Forwarded-Host/Proto). +func (s *Server) webhookPayloadURL(r *http.Request) (url, source string) { + var cfg *config.Config + if s.deps != nil { + cfg = s.deps.Config + } + if cfg != nil { + if u := strings.TrimRight(strings.TrimSpace(cfg.Dashboard.PublicURL), "/"); u != "" { + return u + githubWebhookPath, payloadURLSourceConfig + } + } + s.versionMu.RLock() + pushed := s.hubPushedDashboardURL + s.versionMu.RUnlock() + if cfg != nil && pushed == "" { + pushed = cfg.Hub.DashboardURL + } + if u := strings.TrimRight(strings.TrimSpace(pushed), "/"); u != "" { + return u + githubWebhookPath, payloadURLSourceHub + } + if r != nil { + host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host")) + if host == "" { + host = r.Host + } + if host != "" && !isLoopbackHost(host) { + scheme := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto")) + if scheme == "" { + scheme = "https" + } + return scheme + "://" + host + githubWebhookPath, payloadURLSourceRequest + } + } + return "", payloadURLSourceNone +} + +func isLoopbackHost(host string) bool { + h := strings.ToLower(host) + if i := strings.LastIndex(h, ":"); i > 0 && !strings.Contains(h[i:], "]") { + h = h[:i] + } + return h == "localhost" || h == "127.0.0.1" || h == "[::1]" || h == "::1" +} + +func (s *Server) webhookSetupTargets() (org string, repos []string) { + if s.deps == nil || s.deps.Config == nil { + return "", nil + } + p := s.deps.Config.Project + for _, r := range p.Repos { + r = strings.TrimSpace(r) + if r == "" { + continue + } + if _, name, ok := strings.Cut(r, "/"); ok { + r = name + } + repos = append(repos, r) + } + return strings.TrimSpace(p.Org), repos +} + +func (s *Server) webhookProbeContext(timeout time.Duration) (context.Context, context.CancelFunc) { + ctx := context.Background() + if s.deps != nil && s.deps.Ctx != nil { + ctx = s.deps.Ctx + } + return context.WithTimeout(ctx, timeout) +} + +// handleConfigGitHubWebhooks serves GET /api/config/github/webhooks +// (owner-only). +func (s *Server) handleConfigGitHubWebhooks(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + payloadURL, source := s.webhookPayloadURL(r) + org, repos := s.webhookSetupTargets() + secretSource := ghpkg.WebhookSecretSource() + resp := webhookSetupResponse{ + PayloadURL: payloadURL, + PayloadURLSource: source, + Path: githubWebhookPath, + ContentType: "application/json", + Events: append([]string(nil), ghpkg.WebhookEvents...), + OptionalEvents: ghpkg.WebhookEventsOptional, + Verification: ghpkg.WebhookVerification(), + Hosted: ghpkg.WebhookRelayKey() != "", + SecretConfigured: secretSource != ghpkg.WebhookSecretSourceNone, + SecretSource: secretSource, + CanGenerateSecret: secretSource != ghpkg.WebhookSecretSourceEnv, + Health: ghpkg.WebhookHealthSnapshot(), + Org: org, + Repos: repos, + DocsURL: webhookDocsURL, + } + if secretSource == ghpkg.WebhookSecretSourceFile { + resp.SecretPath = ghpkg.WebhookSecretPath() + } + resp.Health.Verification = resp.Verification + var auth *ghpkg.AppAuth + if s.deps != nil { + auth = s.deps.GHAppAuth + } + if auth != nil && org != "" { + ctx, cancel := s.webhookProbeContext(webhookPermissionProbeTimeout) + info, err := webhookPermissionProbe(ctx, auth) + cancel() + if err != nil { + resp.AppProbeError = err.Error() + } else { + resp.AppOrgHooks, resp.AppRepoHooks = ghpkg.WebhookHookPermissions(info) + } + } + resp.CreateAvailable = resp.SecretConfigured && resp.PayloadURL != "" && (resp.AppOrgHooks || (resp.AppRepoHooks && len(repos) > 0)) + jsonResponse(w, resp) +} + +// handleConfigGitHubWebhooksGenerateSecret serves +// POST /api/config/github/webhooks/secret (owner-only). It mints a secret, +// stores it on the data volume and returns it ONCE; the receiver verifies +// GitHub-signed deliveries with it from the next request on. +func (s *Server) handleConfigGitHubWebhooksGenerateSecret(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + secret, err := ghpkg.GenerateWebhookSecret() + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, ghpkg.ErrWebhookSecretFromEnv) { + status = http.StatusConflict + } + jsonError(w, err.Error(), status) + return + } + user := strings.TrimSpace(r.Header.Get("X-Hive-User")) + s.audit.LogRecord(user, auditWebhookSecretGenerated, auditDetail("path", ghpkg.WebhookSecretPath()), "", "", 0) + s.webhookLogger().Info("github webhook secret generated from the dashboard", "user", user, "path", ghpkg.WebhookSecretPath()) + jsonResponse(w, map[string]any{ + "secret": secret, + "secret_path": ghpkg.WebhookSecretPath(), + "verification": ghpkg.WebhookVerification(), + "note": "Shown once. Paste it into the GitHub webhook's Secret field; the hive keeps only the file.", + }) +} + +type webhookCreateBody struct { + // Scope is "organization" or "repository"; Repo names the repository + // (bare name) for the repository scope. + Scope string `json:"scope"` + Repo string `json:"repo"` +} + +// handleConfigGitHubWebhooksCreate serves POST /api/config/github/webhooks/create +// (owner-only): creates the org or repo hook through the App installation. +func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var body webhookCreateBody + if r.Body != nil { + // An empty body means "organization scope". + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + jsonError(w, "invalid JSON body", http.StatusBadRequest) + return + } + } + secret := ghpkg.WebhookSecret() + if secret == "" { + jsonError(w, "no webhook secret configured — generate one first", http.StatusConflict) + return + } + payloadURL, _ := s.webhookPayloadURL(r) + if payloadURL == "" { + jsonError(w, "this hive has no public dashboard URL; set dashboard.public_url first", http.StatusConflict) + return + } + org, repos := s.webhookSetupTargets() + if org == "" { + jsonError(w, "project.org is not configured", http.StatusConflict) + return + } + var auth *ghpkg.AppAuth + if s.deps != nil { + auth = s.deps.GHAppAuth + } + if auth == nil { + jsonError(w, "GitHub App auth not configured", http.StatusConflict) + return + } + ctx, cancel := s.webhookProbeContext(webhookCreateTimeout) + defer cancel() + info, err := webhookPermissionProbe(ctx, auth) + if err != nil { + jsonError(w, "could not read the App installation's permissions: "+err.Error(), http.StatusBadGateway) + return + } + orgHooks, repoHooks := ghpkg.WebhookHookPermissions(info) + req := ghpkg.WebhookCreateRequest{Org: org, PayloadURL: payloadURL, Secret: secret, Events: ghpkg.WebhookEvents} + switch strings.ToLower(strings.TrimSpace(body.Scope)) { + case "", "organization", "org": + if !orgHooks { + jsonError(w, "the App installation has no organization_hooks: write permission", http.StatusForbidden) + return + } + case "repository", "repo": + if !repoHooks { + jsonError(w, "the App installation has no repository_hooks: write permission", http.StatusForbidden) + return + } + repo := strings.TrimSpace(body.Repo) + if _, name, ok := strings.Cut(repo, "/"); ok { + repo = name + } + if repo == "" || !containsFold(repos, repo) { + jsonError(w, "repo must be one of this hive's project.repos", http.StatusBadRequest) + return + } + req.Repo = repo + default: + jsonError(w, "scope must be organization or repository", http.StatusBadRequest) + return + } + res, err := auth.CreateWebhook(ctx, req) + if err != nil { + status := http.StatusBadGateway + if errors.Is(err, ghpkg.ErrWebhookExists) { + status = http.StatusConflict + } + jsonError(w, err.Error(), status) + return + } + user := strings.TrimSpace(r.Header.Get("X-Hive-User")) + s.audit.LogRecord(user, auditWebhookCreated, auditDetail("scope", res.Scope, "target", res.Target, "payload_url", payloadURL), "", res.Target, 0) + s.webhookLogger().Info("github webhook created through the App", "user", user, "scope", res.Scope, "target", res.Target) + jsonResponse(w, res) +} + +func containsFold(list []string, v string) bool { + for _, s := range list { + if strings.EqualFold(s, v) { + return true + } + } + return false +} diff --git a/src/pkg/dashboard/api_github_webhook_setup_test.go b/src/pkg/dashboard/api_github_webhook_setup_test.go new file mode 100644 index 0000000000..f657d89057 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_setup_test.go @@ -0,0 +1,183 @@ +package dashboard + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +func webhookSetupGet(t *testing.T, s *Server, owner bool, host string) (*httptest.ResponseRecorder, webhookSetupResponse) { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/config/github/webhooks", nil) + if host != "" { + req.Host = host + } + if owner { + markOwnerRequest(req) + } + s.mux.ServeHTTP(rec, req) + var body webhookSetupResponse + if rec.Code == http.StatusOK { + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + } + return rec, body +} + +func TestWebhookSetup_OwnerOnly(t *testing.T) { + srv := newFullServer(t) + rec, _ := webhookSetupGet(t, srv, false, "") + if rec.Code != http.StatusForbidden { + t.Fatalf("non-owner GET: %d, want 403", rec.Code) + } + rec = httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/secret", nil) + srv.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("non-owner generate: %d, want 403", rec.Code) + } +} + +// The GET describes the setup without ever carrying the secret; the generate +// call returns it once and flips the verification mode to github-signed. +func TestWebhookSetup_PayloadURLSecretLifecycle(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "") + restore := ghpkg.SetWebhookSecretPathForTest(filepath.Join(t.TempDir(), "secret.key")) + defer restore() + restoreRelay := ghpkg.SetWebhookRelayKeyPathForTest(filepath.Join(t.TempDir(), "relay.key")) + defer restoreRelay() + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "https://hive.example.org/" + + rec, body := webhookSetupGet(t, srv, true, "") + if rec.Code != http.StatusOK { + t.Fatalf("GET: %d %s", rec.Code, rec.Body.String()) + } + if body.PayloadURL != "https://hive.example.org"+githubWebhookPath { + t.Fatalf("payload_url = %q", body.PayloadURL) + } + if body.PayloadURLSource != payloadURLSourceConfig { + t.Fatalf("payload_url_source = %q", body.PayloadURLSource) + } + if body.SecretConfigured || body.SecretSource != "" || !body.CanGenerateSecret { + t.Fatalf("fresh hive secret state: configured=%v source=%q canGenerate=%v", body.SecretConfigured, body.SecretSource, body.CanGenerateSecret) + } + if body.Verification != ghpkg.WebhookVerificationNone || body.Hosted { + t.Fatalf("verification=%q hosted=%v, want none/false", body.Verification, body.Hosted) + } + if body.CreateAvailable { + t.Fatal("create must not be offered without a secret") + } + if len(body.Events) == 0 || body.Events[0] != "check_run" || body.ContentType != "application/json" { + t.Fatalf("events/content type: %v %q", body.Events, body.ContentType) + } + + rec = httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/secret", nil) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("generate: %d %s", rec.Code, rec.Body.String()) + } + var gen map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &gen) + secret, _ := gen["secret"].(string) + if len(secret) != 64 || secret != ghpkg.WebhookSecret() { + t.Fatalf("generated secret not returned/stored: %q", secret) + } + if gen["verification"] != ghpkg.WebhookVerificationGitHub { + t.Fatalf("verification after generate = %v", gen["verification"]) + } + + rec, body = webhookSetupGet(t, srv, true, "") + if !body.SecretConfigured || body.SecretSource != ghpkg.WebhookSecretSourceFile || body.Verification != ghpkg.WebhookVerificationGitHub { + t.Fatalf("after generate: configured=%v source=%q verification=%q", body.SecretConfigured, body.SecretSource, body.Verification) + } + if strings.Contains(rec.Body.String(), secret) { + t.Fatal("GET must never echo the secret") + } +} + +func TestWebhookSetup_EnvSecretBlocksGenerate(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv := newFullServer(t) + _, body := webhookSetupGet(t, srv, true, "") + if body.SecretSource != ghpkg.WebhookSecretSourceEnv || body.CanGenerateSecret { + t.Fatalf("env secret: source=%q canGenerate=%v", body.SecretSource, body.CanGenerateSecret) + } + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/secret", nil) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusConflict { + t.Fatalf("generate with env secret: %d, want 409", rec.Code) + } +} + +func TestWebhookSetup_PayloadURLFallsBackToRequestHost(t *testing.T) { + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "" + srv.deps.Config.Hub.DashboardURL = "" + _, body := webhookSetupGet(t, srv, true, "hive.dan.example") + if body.PayloadURL != "https://hive.dan.example"+githubWebhookPath || body.PayloadURLSource != payloadURLSourceRequest { + t.Fatalf("payload_url=%q source=%q", body.PayloadURL, body.PayloadURLSource) + } + _, body = webhookSetupGet(t, srv, true, "localhost:8080") + if body.PayloadURL != "" || body.PayloadURLSource != payloadURLSourceNone { + t.Fatalf("loopback host must not be offered: %q %q", body.PayloadURL, body.PayloadURLSource) + } +} + +// Create is offered only when the App holds a hooks write permission, and +// the create endpoint refuses scopes the installation cannot manage. +func TestWebhookSetup_CreateGatedOnAppPermission(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "https://hive.example.org" + srv.deps.Config.Project.Org = "projectbluefin" + srv.deps.Config.Project.Repos = []string{"bluefin", "projectbluefin/common"} + srv.deps.GHAppAuth = &ghpkg.AppAuth{} + orig := webhookPermissionProbe + defer func() { webhookPermissionProbe = orig }() + webhookPermissionProbe = func(context.Context, *ghpkg.AppAuth) (*ghpkg.InstallationInfo, error) { + return &ghpkg.InstallationInfo{Permissions: map[string]string{ghpkg.PermissionRepositoryHooks: "write"}}, nil + } + + _, body := webhookSetupGet(t, srv, true, "") + if body.AppOrgHooks || !body.AppRepoHooks || !body.CreateAvailable { + t.Fatalf("org=%v repo=%v create=%v", body.AppOrgHooks, body.AppRepoHooks, body.CreateAvailable) + } + if len(body.Repos) != 2 || body.Repos[1] != "common" { + t.Fatalf("repos = %v, want bare names", body.Repos) + } + + post := func(payload string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/create", strings.NewReader(payload)) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + return rec + } + if rec := post(`{"scope":"organization"}`); rec.Code != http.StatusForbidden { + t.Fatalf("org scope without organization_hooks: %d %s", rec.Code, rec.Body.String()) + } + if rec := post(`{"scope":"repository","repo":"not-ours"}`); rec.Code != http.StatusBadRequest { + t.Fatalf("unknown repo: %d %s", rec.Code, rec.Body.String()) + } + + webhookPermissionProbe = func(context.Context, *ghpkg.AppAuth) (*ghpkg.InstallationInfo, error) { + return &ghpkg.InstallationInfo{Permissions: map[string]string{}}, nil + } + _, body = webhookSetupGet(t, srv, true, "") + if body.CreateAvailable { + t.Fatal("create offered without any hooks permission") + } +} diff --git a/src/pkg/dashboard/static-src/css/config-webhooks.css b/src/pkg/dashboard/static-src/css/config-webhooks.css new file mode 100644 index 0000000000..e4ee0d2114 --- /dev/null +++ b/src/pkg/dashboard/static-src/css/config-webhooks.css @@ -0,0 +1,27 @@ + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017) */ + .wh-section-title { font-weight: 600; font-size: var(--fs-base); margin: var(--sp-5) 0 var(--sp-2); } + .wh-intro { font-size: var(--fs-sm); color: var(--muted); margin-bottom: var(--sp-3); } + .wh-muted { color: var(--muted); } + div.wh-muted { font-size: var(--fs-sm); } + .wh-small { font-size: var(--fs-sm); } + .wh-warn { color: var(--yellow); } + .wh-ok { color: var(--green); } + .wh-error { font-size: var(--fs-sm); color: var(--red); } + .wh-note { margin-top: var(--sp-2); } + .wh-card { border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-4) var(--sp-5); margin-bottom: var(--sp-5); background: var(--surface); } + .wh-card-head { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; margin-bottom: var(--sp-4); } + .wh-card-title { font-weight: 600; font-size: var(--fs-md); } + .wh-step { font-size: var(--fs-sm); font-weight: 600; margin: var(--sp-2) 0 var(--sp-1); } + .wh-step-gap { margin-top: var(--sp-4); } + .wh-code { font-size: var(--fs-xs); user-select: all; } + .wh-chip { font-size: var(--fs-xs); font-weight: 600; padding: var(--sp-1) var(--sp-4); border-radius: var(--r); border: 1px solid currentColor; } + .wh-chip-muted { color: var(--muted); } + .wh-chip-ok { color: var(--green); } + .wh-chip-warn { color: var(--yellow); } + .wh-actions { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; font-size: var(--fs-sm); } + .wh-actions .hv-btn { margin-left: var(--sp-2); } + .wh-secret-once { border: 1px solid var(--green); border-radius: var(--r); padding: var(--sp-3) var(--sp-4); margin: var(--sp-2) 0; font-size: var(--fs-sm); } + .wh-secret-once-title { font-weight: 600; color: var(--green); } + .wh-secret-value { margin: var(--sp-1) 0; word-break: break-all; } + .wh-create { border-top: 1px solid var(--border); margin-top: var(--sp-4); padding-top: var(--sp-4); } + .gov-stat.wh-clickable { cursor: pointer; } diff --git a/src/pkg/dashboard/static-src/js/config-dialog.js b/src/pkg/dashboard/static-src/js/config-dialog.js index 7622fd4b20..99f72ec85d 100644 --- a/src/pkg/dashboard/static-src/js/config-dialog.js +++ b/src/pkg/dashboard/static-src/js/config-dialog.js @@ -53,6 +53,7 @@ function closeConfigDialog() { _configModalOpen = false; + if (typeof webhookSetupReset === 'function') webhookSetupReset(); // Invalidate any in-flight load so a late reply cannot repopulate a // dialog the operator has already dismissed. _configLoadSeq++; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index b8ca800a08..fd02e566d9 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -49,7 +49,7 @@ } // The Forge App tab renders a shell synchronously, then loads the // credential inventory (fingerprints only) from the spoke. - if (tabId === 'Forge App') loadForgeApps(); + if (tabId === 'Forge App') { loadForgeApps(); loadWebhookSetup(); } // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); @@ -1735,7 +1735,8 @@ when its forge matches the forge this hive's repos live on; everything else is read-only. Private keys appear as fingerprints — key material never leaves the hive. </div> - <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div>`; + <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div> + ${renderGovWebhooksSection()}`; } async function loadForgeApps() { diff --git a/src/pkg/dashboard/static-src/js/config-webhooks.js b/src/pkg/dashboard/static-src/js/config-webhooks.js new file mode 100644 index 0000000000..986ff42f93 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/config-webhooks.js @@ -0,0 +1,217 @@ + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017). + Owner-facing setup for /api/webhook/github: the exact payload URL, + the events to tick, a per-hive secret the owner generates here and + sees once, optional App-side hook creation, and the live receiver + status. Hosted spokes get deliveries through the hub relay with no + owner action, so for them the section is a status readout. */ + + let _webhookSetup = null; + let _webhookSecretOnce = ''; + let _webhookConfirmRegenerate = false; + + function renderGovWebhooksSection() { + return ` + <div class="wh-section-title">GitHub webhooks</div> + <div class="wh-intro"> + Webhooks let GitHub tell this hive about pushes, PRs, reviews and checks as they + happen: faster kicks, fewer API calls, a slower safety-net cadence. + <a href="${webhookDocsHref()}" target="_blank" rel="noopener noreferrer">Owner guide ↗</a> + </div> + <div id="webhook-setup-host"><div class="wh-muted">Loading webhook status…</div></div>`; + } + + function webhookDocsHref() { + return (_webhookSetup && _webhookSetup.docs_url) || 'https://github.com/hivecommons/hive/blob/v5/src/docs/webhooks.md#setting-up-webhooks-owner-guide'; + } + + async function loadWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + if (!host) return; + try { + const res = await fetch('/api/config/github/webhooks'); + if (res.status === 403) { + host.innerHTML = '<div class="wh-muted">Owner access required to view webhook setup.</div>'; + return; + } + if (!res.ok) throw new Error('HTTP ' + res.status); + _webhookSetup = await res.json(); + renderWebhookSetup(); + } catch (e) { + host.innerHTML = `<div class="wh-error">Failed to load webhook status: ${esc(e.message)}</div>`; + } + } + + function webhookStatusChip(d) { + const h = d.health || {}; + if (d.verification === 'none') return chip('not configured', 'muted'); + if (h.events_1h > 0 || h.healthy) return chip(`receiving · ${h.events_1h || 0}/1h`, 'ok'); + if (h.last_event_at) return chip('quiet · last ' + webhookAgo(h.last_event_at), 'warn'); + return chip('no events yet', 'warn'); + function chip(text, tone) { + return `<span class="wh-chip wh-chip-${tone}">${esc(text)}</span>`; + } + } + + function webhookAgo(iso) { + const t = Date.parse(iso); + if (!t) return '—'; + const s = Math.max(0, Math.round((Date.now() - t) / 1000)); + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.round(s / 60) + 'm ago'; + if (s < 86400) return Math.round(s / 3600) + 'h ago'; + return Math.round(s / 86400) + 'd ago'; + } + + function webhookCopyBtn(text, label) { + return `<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCopy" data-arg-types="t" data-copy="${esc(text)}">${esc(label || 'Copy')}</button>`; + } + + async function webhookCopy(btn) { + const text = btn && btn.dataset.copy || ''; + const ok = await writeClipboardText(text); + showToast(ok ? 'Copied' : 'Could not copy — select the text and copy it by hand', ok ? 'info' : 'error'); + } + + function renderWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + const d = _webhookSetup; + if (!host || !d) return; + const h = d.health || {}; + const row = (label, valueHtml) => forgeAppRow(label, valueHtml); + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + const relayHeld = !!d.hosted; + const direct = d.secret_source !== ''; + + let html = `<div class="wh-card"> + <div class="wh-card-head"> + <span class="wh-card-title">Receiver status</span> + ${webhookStatusChip(d)} + </div> + ${row('Verification', code(d.verification || 'none') + (relayHeld && !direct ? ' <span class="wh-muted">— hub relay; the GitHub App installation delivers through the hub, nothing to set up</span>' : ''))} + ${row('Last event', h.last_event_at ? esc(formatDashboardDateTime(h.last_event_at)) + ' <span class="wh-muted">(' + esc(webhookAgo(h.last_event_at)) + ')</span>' : '<span class="wh-muted">none yet</span>')} + ${row('Deliveries (1h)', esc(String(h.events_1h || 0)) + ' <span class="wh-muted">· ' + esc(String(h.invalidations_1h || 0)) + ' cache patches · ' + esc(String(h.ci_events_1h || 0)) + ' CI events</span>')} + ${row('Cadence', code(h.mode || 'polling') + (h.mode_reason ? ' <span class="wh-warn">' + esc(h.mode_reason) + '</span>' : ''))} + ${(!h.last_event_at && d.verification !== 'none') ? '<div class="wh-warn wh-note">No events yet — on GitHub open the webhook (Settings → Webhooks → your hook → Recent deliveries) and check the response code: 401 means the secret differs, 404 means the path is wrong, 503 means this hive has no secret.</div>' : ''} + </div>`; + + html += `<div class="wh-card"> + <div class="wh-card-title wh-card-head">${relayHeld ? 'Direct webhook (optional)' : 'Set up a webhook on GitHub'}</div> + ${relayHeld ? '<div class="wh-muted wh-intro">This hive already receives deliveries the hub relays from the GitHub App installation. A direct webhook with your own secret is only needed for repositories the App is not installed on.</div>' : ''} + <div class="wh-step">1. Secret</div> + ${renderWebhookSecretBlock(d)} + <div class="wh-step wh-step-gap">2. Add the webhook in GitHub</div> + <div class="wh-muted">Organization (one hook for every repo) → <b>Settings → Webhooks → Add webhook</b>, or per repository under the repo's Settings.</div> + ${row('Payload URL', d.payload_url ? code(d.payload_url) + webhookCopyBtn(d.payload_url) + ' <span class="wh-muted">(' + esc(d.payload_url_source) + ')</span>' : '<span class="wh-warn">unknown — set <code>dashboard.public_url</code> to this hive\u2019s public origin; the path is <code>' + esc(d.path) + '</code></span>')} + ${row('Content type', code(d.content_type))} + ${row('Secret', '<span class="wh-muted">the value from step 1</span>')} + ${row('SSL verification', '<span class="wh-muted">keep enabled</span>')} + ${row('Events', '<span class="wh-muted">"Let me select individual events", then tick:</span><br>' + (d.events || []).map(e => code(e)).join(' ') + webhookCopyBtn((d.events || []).join(', '), 'Copy list'))} + ${row('Optional', Object.entries(d.optional_events || {}).map(([k, v]) => code(k) + ' <span class="wh-muted">— ' + esc(v) + '</span>').join('<br>'))} + ${renderWebhookCreateBlock(d)} + </div>`; + host.innerHTML = html; + } + + function renderWebhookSecretBlock(d) { + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + let html = ''; + if (_webhookSecretOnce) { + html += `<div class="wh-secret-once"> + <div class="wh-secret-once-title">New secret — shown once</div> + <div class="wh-secret-value">${code(_webhookSecretOnce)}${webhookCopyBtn(_webhookSecretOnce)}</div> + <div class="wh-muted">Paste it into the GitHub webhook\u2019s <b>Secret</b> field. The hive keeps only the file; closing this dialog forgets the value.</div> + </div>`; + } + if (d.secret_source === 'env') { + html += `<div class="wh-small">Configured from <code>GITHUB_WEBHOOK_SECRET</code> (a Secret mount). Use that same value in GitHub; rotate it in the deployment, not here.</div>`; + return html; + } + const state = d.secret_source === 'file' + ? `Configured · stored at <code>${esc(d.secret_path || '')}</code>${_webhookSecretOnce ? '' : ' (value not shown again — regenerate if lost)'}` + : '<span class="wh-warn">No secret yet — generate one. Without it every delivery is rejected with 503.</span>'; + const label = _webhookConfirmRegenerate ? 'Confirm: replace secret' : (d.secret_source === 'file' ? 'Regenerate secret' : 'Generate secret'); + const cls = _webhookConfirmRegenerate ? 'btn-danger' : 'btn-primary'; + html += `<div class="wh-actions wh-note"> + <span>${state}</span> + <button type="button" class="hv-btn ${cls} btn-sm" data-action="webhookGenerateSecret" data-arg-types="t">${esc(label)}</button> + ${_webhookConfirmRegenerate ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCancelRegenerate">Cancel</button><span class="wh-muted">GitHub hooks using the old secret will start failing with 401 until you update them.</span>' : ''} + </div>`; + return html; + } + + function renderWebhookCreateBlock(d) { + if (d.app_probe_error) { + return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; + } + if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + const scopes = []; + if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); + if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); + const disabled = !d.create_available; + const why = !d.secret_configured ? 'generate a secret first' : (!d.payload_url ? 'set dashboard.public_url first' : ''); + return `<div class="wh-create"> + <div class="wh-step">Or let the App create it</div> + <div class="wh-actions"> + <select id="webhook-create-scope" class="wh-small">${scopes.join('')}</select> + <button type="button" class="hv-btn btn-primary btn-sm" data-action="webhookCreateHook" data-arg-types="t" ${disabled ? 'disabled' : ''}>Create webhook for me</button> + ${why ? '<span class="wh-muted">— ' + esc(why) + '</span>' : '<span class="wh-muted">uses the App installation\u2019s hooks permission; subscribes the events above with this hive\u2019s secret</span>'} + </div> + <div id="webhook-create-result" class="wh-small wh-note"></div> + </div>`; + } + + function webhookCancelRegenerate() { + _webhookConfirmRegenerate = false; + renderWebhookSetup(); + } + + async function webhookGenerateSecret(btn) { + if (_webhookSetup && _webhookSetup.secret_source === 'file' && !_webhookConfirmRegenerate) { + _webhookConfirmRegenerate = true; + renderWebhookSetup(); + return; + } + _webhookConfirmRegenerate = false; + if (btn) btn.disabled = true; + try { + const res = await fetch('/api/config/github/webhooks/secret', { method: 'POST' }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + _webhookSecretOnce = body.secret || ''; + showToast('Webhook secret generated — copy it now, it is shown once', 'info'); + await loadWebhookSetup(); + } catch (e) { + showToast('Could not generate secret: ' + e.message, 'error'); + if (btn) btn.disabled = false; + } + } + + async function webhookCreateHook(btn) { + const sel = document.getElementById('webhook-create-scope'); + const out = document.getElementById('webhook-create-result'); + const v = (sel && sel.value) || 'organization'; + const [scope, repo] = v.split(':'); + if (btn) btn.disabled = true; + if (out) out.textContent = 'Creating…'; + try { + const res = await fetch('/api/config/github/webhooks/create', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scope, repo: repo || '' }) + }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast('Webhook created', 'info'); + } catch (e) { + if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; + } finally { + if (btn) btn.disabled = false; + } + } + + /* Forget the one-time secret whenever the Settings dialog closes so it + never lingers in a re-rendered tab. */ + function webhookSetupReset() { + _webhookSecretOnce = ''; + _webhookConfirmRegenerate = false; + } diff --git a/src/pkg/dashboard/static-src/js/governor-view.js b/src/pkg/dashboard/static-src/js/governor-view.js index 58cd25c87d..0f22bc9556 100644 --- a/src/pkg/dashboard/static-src/js/governor-view.js +++ b/src/pkg/dashboard/static-src/js/governor-view.js @@ -223,6 +223,30 @@ </div>`; } + /* Webhooks tile (hivecommons/hive#18017): whether GitHub deliveries + reach this hive (direct or hub relay) and how many in the last hour. + Owners click through to Settings → Forge App → GitHub webhooks. */ + function webhookGovStatHtml(data) { + const wh = data && data.webhooks; + if (!wh || typeof wh !== 'object') return ''; + const owner = dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + const none = !wh.verification || wh.verification === 'none'; + const n = Number(wh.events_1h || 0); + let val, cls; + if (none) { val = 'off'; cls = ' dead'; } + else if (n > 0 || wh.healthy) { val = n + '/1h'; cls = ''; } + else if (wh.last_event_at) { val = 'quiet'; cls = ' starting'; } + else { val = 'no events'; cls = ' starting'; } + const how = none + ? 'OFF — no webhook secret and no hub relay key, so every delivery is rejected and the hive polls GitHub on the base cadence.' + : `Verification: ${wh.verification}. Cadence: ${wh.mode || 'polling'}${wh.mode_reason ? ' (' + wh.mode_reason + ')' : ''}. ${n} deliveries in the last hour.`; + const tip = 'WEBHOOKS — does GitHub tell this hive about changes as they happen?\n\nWHAT: With webhooks on, pushes, PRs, reviews and checks wake the governor at once, CI failures kick the owning agent, and the safety-net cadence slows to save API calls.\n\nHOW: ' + how + + (owner ? '\n\nClick to set up or check webhooks (Settings → Forge App).' : ''); + const action = owner ? ' data-action="openConfigDialog" data-config-type="governor" data-tab="Forge App"' : ''; + const label = owner && none ? 'webhooks · set up →' : 'webhooks'; + return `<div class="gov-stat${owner ? ' wh-clickable' : ''}"${action} title="${escapeHtml(tip)}"><div class="label">${label}</div><div class="val${cls}">${escapeHtml(val)}</div></div>`; + } + function renderGovernor(gov, cadenceMatrix, data) { const el = document.getElementById('governor'); const isStarting = !gov.active && !gov.mode; @@ -296,6 +320,7 @@ <div class="gov-stat" aria-label="${escapeHtml(prEquationTitle)}"><div class="label">actionable PRs</div><div class="spark-row"><span class="val">${prCount}</span>${prsSpark}</div>${prEquationHTML ? `<div class="overview-kpi-subline">${prEquationHTML}</div>` : ''}</div> <div class="gov-stat" title="${itmTitle}"><div class="label">MTTR</div><div class="spark-row"><span class="val">${formatFixTime(itmMedian)}</span>${itmSpark}</div></div> ${fleetRunStatHtml()} + ${webhookGovStatHtml(data)} <div class="gov-stat" title="${holdTooltip}"><div class="label">on hold</div><div class="spark-row"><span class="val">${holdTotal}</span>${holdSpark}</div></div> ${(() => { // Merge hotspots tile (GET /api/hotspots): files concurrent PRs diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index d6c8e81100..fe1b1cd620 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -22,6 +22,7 @@ css/cost.css css/alerts-toasts.css css/feedback-faq.css css/config-dialog.css +css/config-webhooks.css css/workflows.css css/workflows-editor.css css/workflows-ingest.css @@ -138,6 +139,7 @@ js/config-state.js js/settings-search.js js/config-dialog.js js/config-tabs.js +js/config-webhooks.js js/config-hub.js js/config-agent.js js/config-governor.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 9ed7f34a33..065d33f36e 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5052,6 +5052,33 @@ .config-body { --config-body-pad: 14px; padding: var(--sp-0) var(--config-body-pad) var(--config-body-pad); } .config-field-row, .config-field-row4 { grid-template-columns: 1fr; } } + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017) */ + .wh-section-title { font-weight: 600; font-size: var(--fs-base); margin: var(--sp-5) 0 var(--sp-2); } + .wh-intro { font-size: var(--fs-sm); color: var(--muted); margin-bottom: var(--sp-3); } + .wh-muted { color: var(--muted); } + div.wh-muted { font-size: var(--fs-sm); } + .wh-small { font-size: var(--fs-sm); } + .wh-warn { color: var(--yellow); } + .wh-ok { color: var(--green); } + .wh-error { font-size: var(--fs-sm); color: var(--red); } + .wh-note { margin-top: var(--sp-2); } + .wh-card { border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-4) var(--sp-5); margin-bottom: var(--sp-5); background: var(--surface); } + .wh-card-head { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; margin-bottom: var(--sp-4); } + .wh-card-title { font-weight: 600; font-size: var(--fs-md); } + .wh-step { font-size: var(--fs-sm); font-weight: 600; margin: var(--sp-2) 0 var(--sp-1); } + .wh-step-gap { margin-top: var(--sp-4); } + .wh-code { font-size: var(--fs-xs); user-select: all; } + .wh-chip { font-size: var(--fs-xs); font-weight: 600; padding: var(--sp-1) var(--sp-4); border-radius: var(--r); border: 1px solid currentColor; } + .wh-chip-muted { color: var(--muted); } + .wh-chip-ok { color: var(--green); } + .wh-chip-warn { color: var(--yellow); } + .wh-actions { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; font-size: var(--fs-sm); } + .wh-actions .hv-btn { margin-left: var(--sp-2); } + .wh-secret-once { border: 1px solid var(--green); border-radius: var(--r); padding: var(--sp-3) var(--sp-4); margin: var(--sp-2) 0; font-size: var(--fs-sm); } + .wh-secret-once-title { font-weight: 600; color: var(--green); } + .wh-secret-value { margin: var(--sp-1) 0; word-break: break-all; } + .wh-create { border-top: 1px solid var(--border); margin-top: var(--sp-4); padding-top: var(--sp-4); } + .gov-stat.wh-clickable { cursor: pointer; } /* Settings → Workflows (#17620 phase 3): library, detail, diagram, ingest */ .wf-nav { display: flex; gap: var(--sp-2); margin-bottom: var(--sp-5); border-bottom: var(--line-width) solid var(--border); } .wf-nav-item { background: none; border: none; border-bottom: 2px solid transparent; color: var(--muted); font: inherit; font-size: var(--fs-sm); font-weight: 700; padding: var(--sp-2) var(--sp-4); cursor: pointer; } @@ -32598,6 +32625,30 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> </div>`; } + /* Webhooks tile (hivecommons/hive#18017): whether GitHub deliveries + reach this hive (direct or hub relay) and how many in the last hour. + Owners click through to Settings → Forge App → GitHub webhooks. */ + function webhookGovStatHtml(data) { + const wh = data && data.webhooks; + if (!wh || typeof wh !== 'object') return ''; + const owner = dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + const none = !wh.verification || wh.verification === 'none'; + const n = Number(wh.events_1h || 0); + let val, cls; + if (none) { val = 'off'; cls = ' dead'; } + else if (n > 0 || wh.healthy) { val = n + '/1h'; cls = ''; } + else if (wh.last_event_at) { val = 'quiet'; cls = ' starting'; } + else { val = 'no events'; cls = ' starting'; } + const how = none + ? 'OFF — no webhook secret and no hub relay key, so every delivery is rejected and the hive polls GitHub on the base cadence.' + : `Verification: ${wh.verification}. Cadence: ${wh.mode || 'polling'}${wh.mode_reason ? ' (' + wh.mode_reason + ')' : ''}. ${n} deliveries in the last hour.`; + const tip = 'WEBHOOKS — does GitHub tell this hive about changes as they happen?\n\nWHAT: With webhooks on, pushes, PRs, reviews and checks wake the governor at once, CI failures kick the owning agent, and the safety-net cadence slows to save API calls.\n\nHOW: ' + how + + (owner ? '\n\nClick to set up or check webhooks (Settings → Forge App).' : ''); + const action = owner ? ' data-action="openConfigDialog" data-config-type="governor" data-tab="Forge App"' : ''; + const label = owner && none ? 'webhooks · set up →' : 'webhooks'; + return `<div class="gov-stat${owner ? ' wh-clickable' : ''}"${action} title="${escapeHtml(tip)}"><div class="label">${label}</div><div class="val${cls}">${escapeHtml(val)}</div></div>`; + } + function renderGovernor(gov, cadenceMatrix, data) { const el = document.getElementById('governor'); const isStarting = !gov.active && !gov.mode; @@ -32671,6 +32722,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> <div class="gov-stat" aria-label="${escapeHtml(prEquationTitle)}"><div class="label">actionable PRs</div><div class="spark-row"><span class="val">${prCount}</span>${prsSpark}</div>${prEquationHTML ? `<div class="overview-kpi-subline">${prEquationHTML}</div>` : ''}</div> <div class="gov-stat" title="${itmTitle}"><div class="label">MTTR</div><div class="spark-row"><span class="val">${formatFixTime(itmMedian)}</span>${itmSpark}</div></div> ${fleetRunStatHtml()} + ${webhookGovStatHtml(data)} <div class="gov-stat" title="${holdTooltip}"><div class="label">on hold</div><div class="spark-row"><span class="val">${holdTotal}</span>${holdSpark}</div></div> ${(() => { // Merge hotspots tile (GET /api/hotspots): files concurrent PRs @@ -55947,6 +55999,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> function closeConfigDialog() { _configModalOpen = false; + if (typeof webhookSetupReset === 'function') webhookSetupReset(); // Invalidate any in-flight load so a late reply cannot repopulate a // dialog the operator has already dismissed. _configLoadSeq++; @@ -56533,7 +56586,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> } // The Forge App tab renders a shell synchronously, then loads the // credential inventory (fingerprints only) from the spoke. - if (tabId === 'Forge App') loadForgeApps(); + if (tabId === 'Forge App') { loadForgeApps(); loadWebhookSetup(); } // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); @@ -58219,7 +58272,8 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> when its forge matches the forge this hive's repos live on; everything else is read-only. Private keys appear as fingerprints — key material never leaves the hive. </div> - <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div>`; + <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div> + ${renderGovWebhooksSection()}`; } async function loadForgeApps() { @@ -59433,6 +59487,223 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> function contribRepoMatchingDisabledEntries(repoFull, disabled) { return (disabled || []).filter(function(entry) { return contribRepoDisabledEntryMatches(repoFull, entry); }); } + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017). + Owner-facing setup for /api/webhook/github: the exact payload URL, + the events to tick, a per-hive secret the owner generates here and + sees once, optional App-side hook creation, and the live receiver + status. Hosted spokes get deliveries through the hub relay with no + owner action, so for them the section is a status readout. */ + + let _webhookSetup = null; + let _webhookSecretOnce = ''; + let _webhookConfirmRegenerate = false; + + function renderGovWebhooksSection() { + return ` + <div class="wh-section-title">GitHub webhooks</div> + <div class="wh-intro"> + Webhooks let GitHub tell this hive about pushes, PRs, reviews and checks as they + happen: faster kicks, fewer API calls, a slower safety-net cadence. + <a href="${webhookDocsHref()}" target="_blank" rel="noopener noreferrer">Owner guide ↗</a> + </div> + <div id="webhook-setup-host"><div class="wh-muted">Loading webhook status…</div></div>`; + } + + function webhookDocsHref() { + return (_webhookSetup && _webhookSetup.docs_url) || 'https://github.com/hivecommons/hive/blob/v5/src/docs/webhooks.md#setting-up-webhooks-owner-guide'; + } + + async function loadWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + if (!host) return; + try { + const res = await fetch('/api/config/github/webhooks'); + if (res.status === 403) { + host.innerHTML = '<div class="wh-muted">Owner access required to view webhook setup.</div>'; + return; + } + if (!res.ok) throw new Error('HTTP ' + res.status); + _webhookSetup = await res.json(); + renderWebhookSetup(); + } catch (e) { + host.innerHTML = `<div class="wh-error">Failed to load webhook status: ${esc(e.message)}</div>`; + } + } + + function webhookStatusChip(d) { + const h = d.health || {}; + if (d.verification === 'none') return chip('not configured', 'muted'); + if (h.events_1h > 0 || h.healthy) return chip(`receiving · ${h.events_1h || 0}/1h`, 'ok'); + if (h.last_event_at) return chip('quiet · last ' + webhookAgo(h.last_event_at), 'warn'); + return chip('no events yet', 'warn'); + function chip(text, tone) { + return `<span class="wh-chip wh-chip-${tone}">${esc(text)}</span>`; + } + } + + function webhookAgo(iso) { + const t = Date.parse(iso); + if (!t) return '—'; + const s = Math.max(0, Math.round((Date.now() - t) / 1000)); + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.round(s / 60) + 'm ago'; + if (s < 86400) return Math.round(s / 3600) + 'h ago'; + return Math.round(s / 86400) + 'd ago'; + } + + function webhookCopyBtn(text, label) { + return `<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCopy" data-arg-types="t" data-copy="${esc(text)}">${esc(label || 'Copy')}</button>`; + } + + async function webhookCopy(btn) { + const text = btn && btn.dataset.copy || ''; + const ok = await writeClipboardText(text); + showToast(ok ? 'Copied' : 'Could not copy — select the text and copy it by hand', ok ? 'info' : 'error'); + } + + function renderWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + const d = _webhookSetup; + if (!host || !d) return; + const h = d.health || {}; + const row = (label, valueHtml) => forgeAppRow(label, valueHtml); + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + const relayHeld = !!d.hosted; + const direct = d.secret_source !== ''; + + let html = `<div class="wh-card"> + <div class="wh-card-head"> + <span class="wh-card-title">Receiver status</span> + ${webhookStatusChip(d)} + </div> + ${row('Verification', code(d.verification || 'none') + (relayHeld && !direct ? ' <span class="wh-muted">— hub relay; the GitHub App installation delivers through the hub, nothing to set up</span>' : ''))} + ${row('Last event', h.last_event_at ? esc(formatDashboardDateTime(h.last_event_at)) + ' <span class="wh-muted">(' + esc(webhookAgo(h.last_event_at)) + ')</span>' : '<span class="wh-muted">none yet</span>')} + ${row('Deliveries (1h)', esc(String(h.events_1h || 0)) + ' <span class="wh-muted">· ' + esc(String(h.invalidations_1h || 0)) + ' cache patches · ' + esc(String(h.ci_events_1h || 0)) + ' CI events</span>')} + ${row('Cadence', code(h.mode || 'polling') + (h.mode_reason ? ' <span class="wh-warn">' + esc(h.mode_reason) + '</span>' : ''))} + ${(!h.last_event_at && d.verification !== 'none') ? '<div class="wh-warn wh-note">No events yet — on GitHub open the webhook (Settings → Webhooks → your hook → Recent deliveries) and check the response code: 401 means the secret differs, 404 means the path is wrong, 503 means this hive has no secret.</div>' : ''} + </div>`; + + html += `<div class="wh-card"> + <div class="wh-card-title wh-card-head">${relayHeld ? 'Direct webhook (optional)' : 'Set up a webhook on GitHub'}</div> + ${relayHeld ? '<div class="wh-muted wh-intro">This hive already receives deliveries the hub relays from the GitHub App installation. A direct webhook with your own secret is only needed for repositories the App is not installed on.</div>' : ''} + <div class="wh-step">1. Secret</div> + ${renderWebhookSecretBlock(d)} + <div class="wh-step wh-step-gap">2. Add the webhook in GitHub</div> + <div class="wh-muted">Organization (one hook for every repo) → <b>Settings → Webhooks → Add webhook</b>, or per repository under the repo's Settings.</div> + ${row('Payload URL', d.payload_url ? code(d.payload_url) + webhookCopyBtn(d.payload_url) + ' <span class="wh-muted">(' + esc(d.payload_url_source) + ')</span>' : '<span class="wh-warn">unknown — set <code>dashboard.public_url</code> to this hive\u2019s public origin; the path is <code>' + esc(d.path) + '</code></span>')} + ${row('Content type', code(d.content_type))} + ${row('Secret', '<span class="wh-muted">the value from step 1</span>')} + ${row('SSL verification', '<span class="wh-muted">keep enabled</span>')} + ${row('Events', '<span class="wh-muted">"Let me select individual events", then tick:</span><br>' + (d.events || []).map(e => code(e)).join(' ') + webhookCopyBtn((d.events || []).join(', '), 'Copy list'))} + ${row('Optional', Object.entries(d.optional_events || {}).map(([k, v]) => code(k) + ' <span class="wh-muted">— ' + esc(v) + '</span>').join('<br>'))} + ${renderWebhookCreateBlock(d)} + </div>`; + host.innerHTML = html; + } + + function renderWebhookSecretBlock(d) { + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + let html = ''; + if (_webhookSecretOnce) { + html += `<div class="wh-secret-once"> + <div class="wh-secret-once-title">New secret — shown once</div> + <div class="wh-secret-value">${code(_webhookSecretOnce)}${webhookCopyBtn(_webhookSecretOnce)}</div> + <div class="wh-muted">Paste it into the GitHub webhook\u2019s <b>Secret</b> field. The hive keeps only the file; closing this dialog forgets the value.</div> + </div>`; + } + if (d.secret_source === 'env') { + html += `<div class="wh-small">Configured from <code>GITHUB_WEBHOOK_SECRET</code> (a Secret mount). Use that same value in GitHub; rotate it in the deployment, not here.</div>`; + return html; + } + const state = d.secret_source === 'file' + ? `Configured · stored at <code>${esc(d.secret_path || '')}</code>${_webhookSecretOnce ? '' : ' (value not shown again — regenerate if lost)'}` + : '<span class="wh-warn">No secret yet — generate one. Without it every delivery is rejected with 503.</span>'; + const label = _webhookConfirmRegenerate ? 'Confirm: replace secret' : (d.secret_source === 'file' ? 'Regenerate secret' : 'Generate secret'); + const cls = _webhookConfirmRegenerate ? 'btn-danger' : 'btn-primary'; + html += `<div class="wh-actions wh-note"> + <span>${state}</span> + <button type="button" class="hv-btn ${cls} btn-sm" data-action="webhookGenerateSecret" data-arg-types="t">${esc(label)}</button> + ${_webhookConfirmRegenerate ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCancelRegenerate">Cancel</button><span class="wh-muted">GitHub hooks using the old secret will start failing with 401 until you update them.</span>' : ''} + </div>`; + return html; + } + + function renderWebhookCreateBlock(d) { + if (d.app_probe_error) { + return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; + } + if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + const scopes = []; + if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); + if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); + const disabled = !d.create_available; + const why = !d.secret_configured ? 'generate a secret first' : (!d.payload_url ? 'set dashboard.public_url first' : ''); + return `<div class="wh-create"> + <div class="wh-step">Or let the App create it</div> + <div class="wh-actions"> + <select id="webhook-create-scope" class="wh-small">${scopes.join('')}</select> + <button type="button" class="hv-btn btn-primary btn-sm" data-action="webhookCreateHook" data-arg-types="t" ${disabled ? 'disabled' : ''}>Create webhook for me</button> + ${why ? '<span class="wh-muted">— ' + esc(why) + '</span>' : '<span class="wh-muted">uses the App installation\u2019s hooks permission; subscribes the events above with this hive\u2019s secret</span>'} + </div> + <div id="webhook-create-result" class="wh-small wh-note"></div> + </div>`; + } + + function webhookCancelRegenerate() { + _webhookConfirmRegenerate = false; + renderWebhookSetup(); + } + + async function webhookGenerateSecret(btn) { + if (_webhookSetup && _webhookSetup.secret_source === 'file' && !_webhookConfirmRegenerate) { + _webhookConfirmRegenerate = true; + renderWebhookSetup(); + return; + } + _webhookConfirmRegenerate = false; + if (btn) btn.disabled = true; + try { + const res = await fetch('/api/config/github/webhooks/secret', { method: 'POST' }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + _webhookSecretOnce = body.secret || ''; + showToast('Webhook secret generated — copy it now, it is shown once', 'info'); + await loadWebhookSetup(); + } catch (e) { + showToast('Could not generate secret: ' + e.message, 'error'); + if (btn) btn.disabled = false; + } + } + + async function webhookCreateHook(btn) { + const sel = document.getElementById('webhook-create-scope'); + const out = document.getElementById('webhook-create-result'); + const v = (sel && sel.value) || 'organization'; + const [scope, repo] = v.split(':'); + if (btn) btn.disabled = true; + if (out) out.textContent = 'Creating…'; + try { + const res = await fetch('/api/config/github/webhooks/create', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scope, repo: repo || '' }) + }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast('Webhook created', 'info'); + } catch (e) { + if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; + } finally { + if (btn) btn.disabled = false; + } + } + + /* Forget the one-time secret whenever the Settings dialog closes so it + never lingers in a re-rendered tab. */ + function webhookSetupReset() { + _webhookSecretOnce = ''; + _webhookConfirmRegenerate = false; + } function renderGovHub(hub) { const dashUrl = hub.dashboard_url || window.location.origin || ''; const snapUrl = hub.snapshot_url || (hub.auto_snapshot && dashUrl ? dashUrl.replace(/\/$/, '') + '/snapshot' : ''); diff --git a/src/pkg/github/app.go b/src/pkg/github/app.go index f68664c810..5ac633d0cc 100644 --- a/src/pkg/github/app.go +++ b/src/pkg/github/app.go @@ -866,7 +866,9 @@ func installationPermissionMap(p *gh.InstallationPermissions) map[string]string put("contents", p.GetContents()) put("issues", p.GetIssues()) put("metadata", p.GetMetadata()) + put(PermissionOrganizationHooks, p.GetOrganizationHooks()) put("pull_requests", p.GetPullRequests()) + put(PermissionRepositoryHooks, p.GetRepositoryHooks()) put("secrets", p.GetSecrets()) put("vulnerability_alerts", p.GetVulnerabilityAlerts()) put("workflows", p.GetWorkflows()) diff --git a/src/pkg/github/webhook_relay_key.go b/src/pkg/github/webhook_relay_key.go index d850abdb66..7d7cc41278 100644 --- a/src/pkg/github/webhook_relay_key.go +++ b/src/pkg/github/webhook_relay_key.go @@ -28,7 +28,8 @@ const WebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" // Values of /api/status.webhooks.verification: how this spoke authenticates // GitHub webhook deliveries. const ( - // WebhookVerificationGitHub: GITHUB_WEBHOOK_SECRET is set and GitHub's + // WebhookVerificationGitHub: a webhook secret is configured (WebhookSecret: + // GITHUB_WEBHOOK_SECRET or the generated secret file) and GitHub's // X-Hub-Signature-256 is verified (self-hosted, or a hub relay to a spoke // that holds the secret). WebhookVerificationGitHub = "github-signed" @@ -109,7 +110,7 @@ func WebhookRelayKeyFingerprint() string { // (WebhookVerification* above). Only the mode ever leaves this function, never // the secret or the key. func WebhookVerification() string { - if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + if WebhookSecret() != "" { return WebhookVerificationGitHub } if WebhookRelayKey() != "" { diff --git a/src/pkg/github/webhook_secret.go b/src/pkg/github/webhook_secret.go new file mode 100644 index 0000000000..49d074b16a --- /dev/null +++ b/src/pkg/github/webhook_secret.go @@ -0,0 +1,133 @@ +package github + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "sync" +) + +// GitHub webhook secret for a self-hosted spoke (hivecommons/hive#18017). +// +// A self-hosted hive verifies X-Hub-Signature-256 on /api/webhook/github +// with a secret the owner also pastes into the GitHub webhook form. It comes +// from GITHUB_WEBHOOK_SECRET when the operator mounts one, otherwise from a +// 0600 file on the data volume that the dashboard's "Generate secret" button +// writes (same pattern as the hub relay key, webhook_relay_key.go). The env +// var always wins: a Secret mount is the deployment's source of truth and the +// dashboard must not pretend it can rotate it. + +const defaultWebhookSecretPath = "/data/github-webhook-secret.key" + +// Values of WebhookSecretSource: where the active webhook secret comes from. +const ( + WebhookSecretSourceEnv = "env" + WebhookSecretSourceFile = "file" + WebhookSecretSourceNone = "" +) + +// webhookSecretBytes is the entropy of a generated secret (256 bits, hex). +const webhookSecretBytes = 32 + +// ErrWebhookSecretFromEnv is returned by GenerateWebhookSecret when +// GITHUB_WEBHOOK_SECRET is set: the file would never be consulted, so writing +// one would only mislead the owner about which secret is active. +var ErrWebhookSecretFromEnv = errors.New("webhook secret is set by " + WebhookSecretEnvVar + "; rotate it there") + +var webhookSecretState = struct { + mu sync.Mutex + path string + secret string + loaded bool +}{path: defaultWebhookSecretPath} + +func webhookSecretFileLocked() string { + st := &webhookSecretState + if !st.loaded { + st.loaded = true + if data, err := os.ReadFile(st.path); err == nil { + st.secret = strings.TrimSpace(string(data)) + } + } + return st.secret +} + +// WebhookSecret is the HMAC secret this spoke verifies GitHub-signed +// deliveries with: GITHUB_WEBHOOK_SECRET, else the generated secret file, else +// "". +func WebhookSecret() string { + if v := strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)); v != "" { + return v + } + webhookSecretState.mu.Lock() + defer webhookSecretState.mu.Unlock() + return webhookSecretFileLocked() +} + +// WebhookSecretSource reports where WebhookSecret comes from +// (WebhookSecretSource*). Never the secret itself. +func WebhookSecretSource() string { + if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + return WebhookSecretSourceEnv + } + if WebhookSecret() != "" { + return WebhookSecretSourceFile + } + return WebhookSecretSourceNone +} + +// WebhookSecretPath is the file a generated secret is persisted to. +func WebhookSecretPath() string { + webhookSecretState.mu.Lock() + defer webhookSecretState.mu.Unlock() + return webhookSecretState.path +} + +// GenerateWebhookSecret mints a fresh random secret, persists it 0600 to the +// secret file (replacing any previous generated secret) and returns it. This +// is the only time the value leaves the hive; callers show it once. It +// refuses with ErrWebhookSecretFromEnv when the env var is set. +func GenerateWebhookSecret() (string, error) { + if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + return "", ErrWebhookSecretFromEnv + } + b := make([]byte, webhookSecretBytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + secret := hex.EncodeToString(b) + st := &webhookSecretState + st.mu.Lock() + defer st.mu.Unlock() + if err := os.MkdirAll(filepath.Dir(st.path), 0o700); err != nil { + return "", err + } + tmp := st.path + ".tmp" + if err := os.WriteFile(tmp, []byte(secret+"\n"), 0o600); err != nil { + return "", err + } + if err := os.Rename(tmp, st.path); err != nil { + return "", err + } + st.secret = secret + st.loaded = true + return secret, nil +} + +// SetWebhookSecretPathForTest points the secret store at path, forgets any +// secret held in memory, and returns a func that restores the previous state. +func SetWebhookSecretPathForTest(path string) func() { + st := &webhookSecretState + st.mu.Lock() + prevPath, prevSecret, prevLoaded := st.path, st.secret, st.loaded + st.path, st.secret, st.loaded = path, "", false + st.mu.Unlock() + return func() { + st.mu.Lock() + st.path, st.secret, st.loaded = prevPath, prevSecret, prevLoaded + st.mu.Unlock() + } +} diff --git a/src/pkg/github/webhook_secret_test.go b/src/pkg/github/webhook_secret_test.go new file mode 100644 index 0000000000..7abdc64ad9 --- /dev/null +++ b/src/pkg/github/webhook_secret_test.go @@ -0,0 +1,105 @@ +package github + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWebhookSecret_FileFallbackAndGenerate(t *testing.T) { + t.Setenv(WebhookSecretEnvVar, "") + dir := t.TempDir() + restore := SetWebhookSecretPathForTest(filepath.Join(dir, "nested", "secret.key")) + defer restore() + + if got := WebhookSecret(); got != "" { + t.Fatalf("no file yet: WebhookSecret() = %q, want empty", got) + } + if got := WebhookSecretSource(); got != WebhookSecretSourceNone { + t.Fatalf("source = %q, want none", got) + } + if got := WebhookVerification(); got == WebhookVerificationGitHub { + t.Fatalf("verification = %q before any secret", got) + } + + secret, err := GenerateWebhookSecret() + if err != nil { + t.Fatalf("GenerateWebhookSecret: %v", err) + } + if len(secret) != webhookSecretBytes*2 { + t.Fatalf("secret length %d, want %d hex chars", len(secret), webhookSecretBytes*2) + } + if got := WebhookSecret(); got != secret { + t.Fatalf("WebhookSecret() = %q, want the generated value", got) + } + if got := WebhookSecretSource(); got != WebhookSecretSourceFile { + t.Fatalf("source = %q, want file", got) + } + if got := WebhookVerification(); got != WebhookVerificationGitHub { + t.Fatalf("verification = %q, want %q", got, WebhookVerificationGitHub) + } + st, err := os.Stat(WebhookSecretPath()) + if err != nil { + t.Fatalf("stat secret file: %v", err) + } + if perm := st.Mode().Perm(); perm != 0o600 { + t.Fatalf("secret file mode %o, want 0600", perm) + } + + // A fresh process reads the persisted value back. + restore2 := SetWebhookSecretPathForTest(WebhookSecretPath()) + defer restore2() + if got := WebhookSecret(); got != secret { + t.Fatalf("reloaded secret = %q, want %q", got, secret) + } + + // Regenerating replaces it. + next, err := GenerateWebhookSecret() + if err != nil || next == secret { + t.Fatalf("regenerate: err=%v same=%v", err, next == secret) + } +} + +func TestWebhookSecret_EnvWinsAndBlocksGenerate(t *testing.T) { + restore := SetWebhookSecretPathForTest(filepath.Join(t.TempDir(), "secret.key")) + defer restore() + if _, err := GenerateWebhookSecret(); err != nil { + t.Fatalf("seed file secret: %v", err) + } + t.Setenv(WebhookSecretEnvVar, "from-env") + if got := WebhookSecret(); got != "from-env" { + t.Fatalf("WebhookSecret() = %q, want env value", got) + } + if got := WebhookSecretSource(); got != WebhookSecretSourceEnv { + t.Fatalf("source = %q, want env", got) + } + if _, err := GenerateWebhookSecret(); err != ErrWebhookSecretFromEnv { + t.Fatalf("GenerateWebhookSecret with env set: err=%v, want ErrWebhookSecretFromEnv", err) + } +} + +func TestWebhookHookPermissions(t *testing.T) { + org, repo := WebhookHookPermissions(nil) + if org || repo { + t.Fatal("nil info must grant nothing") + } + info := &InstallationInfo{Permissions: map[string]string{PermissionOrganizationHooks: "write", PermissionRepositoryHooks: "read"}} + org, repo = WebhookHookPermissions(info) + if !org || repo { + t.Fatalf("org=%v repo=%v, want org write only", org, repo) + } +} + +func TestWebBaseURLForAPI(t *testing.T) { + cases := map[string]string{ + "": "https://github.com", + "https://api.github.com": "https://github.com", + "https://ghe.example.com/api/v3": "https://ghe.example.com", + "https://ghe.example.com/api/v3/": "https://ghe.example.com", + } + for in, want := range cases { + if got := webBaseURLForAPI(in); got != want { + t.Errorf("webBaseURLForAPI(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/src/pkg/github/webhook_setup.go b/src/pkg/github/webhook_setup.go new file mode 100644 index 0000000000..78aaeda11f --- /dev/null +++ b/src/pkg/github/webhook_setup.go @@ -0,0 +1,146 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + gh "github.com/google/go-github/v72/github" +) + +// Owner-facing webhook setup (hivecommons/hive#18017): the events a hive +// consumes on /api/webhook/github and the App-side hook creation the +// dashboard offers when the installation may manage hooks. + +// WebhookEvents is the list of GitHub webhook events the spoke receiver acts +// on, in the order the GitHub webhook form lists them. It is what the +// Settings card tells an owner to tick, and what CreateWebhook subscribes. +// projects_v2_item is organization-scoped and only useful with the +// github_projects work source, so it is listed separately +// (WebhookEventsOptional). +var WebhookEvents = []string{ + "check_run", + "check_suite", + "issue_comment", + "issues", + "pull_request", + "pull_request_review", + "pull_request_review_comment", + "push", + "status", + "workflow_run", +} + +// WebhookEventsOptional are events the receiver understands but that only +// matter with a specific feature enabled. +var WebhookEventsOptional = map[string]string{ + "workflow_job": "CI job counts from the job payload (needs Actions: read on the App)", + "projects_v2_item": "GitHub Projects work source (governor.work_source.type: github_projects); organization webhook only", +} + +// Installation permissions that allow the App to create hooks. +const ( + PermissionOrganizationHooks = "organization_hooks" + PermissionRepositoryHooks = "repository_hooks" +) + +// WebhookHookPermissions reports which hook-management permissions the +// installation holds at write level: org-level hooks and repo-level hooks. +func WebhookHookPermissions(info *InstallationInfo) (orgHooks, repoHooks bool) { + if info == nil { + return false, false + } + return info.Granted(PermissionOrganizationHooks, "write"), info.Granted(PermissionRepositoryHooks, "write") +} + +// ErrWebhookExists is returned when GitHub already has a hook for the same +// payload URL at that scope (GitHub answers 422 "Hook already exists"). +var ErrWebhookExists = errors.New("a webhook for this payload URL already exists") + +// WebhookCreateRequest describes the hook CreateWebhook should create. +type WebhookCreateRequest struct { + // Org is the owner login; Repo is empty for an organization hook. + Org, Repo string + // PayloadURL is the spoke's public receiver URL; Secret the HMAC secret. + PayloadURL, Secret string + Events []string +} + +// WebhookCreateResult names the hook GitHub created. +type WebhookCreateResult struct { + HookID int64 `json:"hook_id"` + Scope string `json:"scope"` // "organization" or "repository" + Target string `json:"target"` + URL string `json:"url"` // GitHub settings page for the hook +} + +// CreateWebhook creates an organization hook (req.Repo == "") or a repository +// hook through the App installation token. The token carries every granted +// permission, so the caller checks WebhookHookPermissions first and only +// offers the action when the matching write permission is held. +func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) (*WebhookCreateResult, error) { + if a == nil { + return nil, errors.New("GitHub App auth not configured") + } + if strings.TrimSpace(req.Org) == "" || strings.TrimSpace(req.PayloadURL) == "" || strings.TrimSpace(req.Secret) == "" { + return nil, errors.New("org, payload URL and secret are required") + } + token, err := a.Token(ctx) + if err != nil { + return nil, fmt.Errorf("minting installation token: %w", err) + } + client := newTokenClient(token, a.apiURL) + events := req.Events + if len(events) == 0 { + events = WebhookEvents + } + hook := &gh.Hook{ + Active: gh.Ptr(true), + Events: events, + Config: &gh.HookConfig{ + URL: gh.Ptr(req.PayloadURL), + ContentType: gh.Ptr("json"), + Secret: gh.Ptr(req.Secret), + InsecureSSL: gh.Ptr("0"), + }, + } + var ( + created *gh.Hook + resp *gh.Response + ) + res := &WebhookCreateResult{} + if req.Repo == "" { + created, resp, err = client.Organizations.CreateHook(ctx, req.Org, hook) + res.Scope, res.Target = "organization", req.Org + } else { + created, resp, err = client.Repositories.CreateHook(ctx, req.Org, req.Repo, hook) + res.Scope, res.Target = "repository", req.Org+"/"+req.Repo + } + if err != nil { + if resp != nil && resp.StatusCode == http.StatusUnprocessableEntity && strings.Contains(strings.ToLower(err.Error()), "already exists") { + return nil, ErrWebhookExists + } + return nil, err + } + res.HookID = created.GetID() + base := strings.TrimSuffix(webBaseURLForAPI(a.apiURL), "/") + if req.Repo == "" { + res.URL = fmt.Sprintf("%s/organizations/%s/settings/hooks/%d", base, req.Org, res.HookID) + } else { + res.URL = fmt.Sprintf("%s/%s/%s/settings/hooks/%d", base, req.Org, req.Repo, res.HookID) + } + return res, nil +} + +// webBaseURLForAPI maps an API URL to the web origin a hook's settings page +// lives on: api.github.com → github.com, https://ghe.example/api/v3 → +// https://ghe.example. +func webBaseURLForAPI(apiURL string) string { + u := strings.TrimSpace(apiURL) + if u == "" || strings.Contains(u, "api.github.com") { + return "https://github.com" + } + return strings.TrimSuffix(strings.TrimSuffix(u, "/"), "/api/v3") +} From 77e753650425230581815bd385dcb4622186a253 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 23:56:24 +0000 Subject: [PATCH 129/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.239.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.239.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 13 +++++++++++++ changelog.d/added-17987-review-perspsrc-resolver.md | 1 - ...fixed-activity-monitor-workflow-column-header.md | 1 - changelog.d/fixed-canary-kick-refusal.md | 1 - changelog.d/fixed-contributor-sweep-cancellation.md | 1 - .../fixed-spoke-installs-missing-startup-probe.md | 1 - 6 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-17987-review-perspsrc-resolver.md delete mode 100644 changelog.d/fixed-activity-monitor-workflow-column-header.md delete mode 100644 changelog.d/fixed-canary-kick-refusal.md delete mode 100644 changelog.d/fixed-contributor-sweep-cancellation.md delete mode 100644 changelog.d/fixed-spoke-installs-missing-startup-probe.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bf271ee566..6883be0e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.239.0) + +### Added + +- **Resolver for `review.perspectives_source`** that fetches a `kind: ReviewPerspectives` document behind the same seed-only allowlist as `definition_source`, and keeps the last-known-good set with a visible warning when a later fetch fails or a document no longer validates, instead of silently falling back to the built-in perspectives. The review relay, dispatch, head aggregation and `${REVIEW_PERSPECTIVES}` now read the resolved set via `ReviewConfig.EffectivePerspectives()`; with no `perspectives_source` configured, behaviour is unchanged. ([#17987](https://github.com/hivecommons/hive/issues/17987), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) + +### Fixed + +- Activity monitor: the column header now follows the workflow model the rows wear — a repository bound to a custom model gets that model's column names over its chevrons (ungrouped pages are partitioned by model with a header per run), instead of hive-default's "PR in progress / Merge queue" names (#18002). +- Kicks are no longer refused wholesale as "prompt injection" because of the ioscan exfiltration canary: the preamble is now a labelled `=== HIVE HARNESS HEADER` block that states it is the operator's own configuration and not a task (Claude Sonnet 5 via the Copilot CLI refused ~65% of one hosted supervisor's kicks on the old `SECURITY CANARY:` marker), and a kick that is still refused with a canary in front is re-delivered once without it instead of burning the cadence slot (#18012). +- Stop contributor automerge passes cleanly when a client hand-over or shutdown cancels an in-flight sweep, without filing a spurious failure report. +- Spoke self-upgrade now installs the hosted template's startupProbe (10-min budget) on a hive Deployment that has none, not only raises an undersized one — pre-#9692 spokes on pull-only clusters were liveness-killed 90s into a forge-blocked boot and crash-looped (grey dot) for the length of the GHE outage; the fleet status-dot hovers now carry a colour legend and the grey dot says "no heartbeat in 5 min". + ## 2026-11-12 (v5.238.1) ### Fixed diff --git a/changelog.d/added-17987-review-perspsrc-resolver.md b/changelog.d/added-17987-review-perspsrc-resolver.md deleted file mode 100644 index f3bbd4c8eb..0000000000 --- a/changelog.d/added-17987-review-perspsrc-resolver.md +++ /dev/null @@ -1 +0,0 @@ -- **Resolver for `review.perspectives_source`** that fetches a `kind: ReviewPerspectives` document behind the same seed-only allowlist as `definition_source`, and keeps the last-known-good set with a visible warning when a later fetch fails or a document no longer validates, instead of silently falling back to the built-in perspectives. The review relay, dispatch, head aggregation and `${REVIEW_PERSPECTIVES}` now read the resolved set via `ReviewConfig.EffectivePerspectives()`; with no `perspectives_source` configured, behaviour is unchanged. ([#17987](https://github.com/hivecommons/hive/issues/17987), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-activity-monitor-workflow-column-header.md b/changelog.d/fixed-activity-monitor-workflow-column-header.md deleted file mode 100644 index e868548c15..0000000000 --- a/changelog.d/fixed-activity-monitor-workflow-column-header.md +++ /dev/null @@ -1 +0,0 @@ -- Activity monitor: the column header now follows the workflow model the rows wear — a repository bound to a custom model gets that model's column names over its chevrons (ungrouped pages are partitioned by model with a header per run), instead of hive-default's "PR in progress / Merge queue" names (#18002). diff --git a/changelog.d/fixed-canary-kick-refusal.md b/changelog.d/fixed-canary-kick-refusal.md deleted file mode 100644 index 5c1235ffe5..0000000000 --- a/changelog.d/fixed-canary-kick-refusal.md +++ /dev/null @@ -1 +0,0 @@ -- Kicks are no longer refused wholesale as "prompt injection" because of the ioscan exfiltration canary: the preamble is now a labelled `=== HIVE HARNESS HEADER` block that states it is the operator's own configuration and not a task (Claude Sonnet 5 via the Copilot CLI refused ~65% of one hosted supervisor's kicks on the old `SECURITY CANARY:` marker), and a kick that is still refused with a canary in front is re-delivered once without it instead of burning the cadence slot (#18012). diff --git a/changelog.d/fixed-contributor-sweep-cancellation.md b/changelog.d/fixed-contributor-sweep-cancellation.md deleted file mode 100644 index be00f6d940..0000000000 --- a/changelog.d/fixed-contributor-sweep-cancellation.md +++ /dev/null @@ -1 +0,0 @@ -- Stop contributor automerge passes cleanly when a client hand-over or shutdown cancels an in-flight sweep, without filing a spurious failure report. diff --git a/changelog.d/fixed-spoke-installs-missing-startup-probe.md b/changelog.d/fixed-spoke-installs-missing-startup-probe.md deleted file mode 100644 index 3680f325a5..0000000000 --- a/changelog.d/fixed-spoke-installs-missing-startup-probe.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke self-upgrade now installs the hosted template's startupProbe (10-min budget) on a hive Deployment that has none, not only raises an undersized one — pre-#9692 spokes on pull-only clusters were liveness-killed 90s into a forge-blocked boot and crash-looped (grey dot) for the length of the GHE outage; the fleet status-dot hovers now carry a colour legend and the grey dot says "no heartbeat in 5 min". From 3d553e32ef5347d9d1ba9a0e499c0d7459abbd60 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 00:27:24 +0000 Subject: [PATCH 130/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.240.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.240.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/added-reference-repos.md | 1 - changelog.d/added-webhook-owner-setup.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-reference-repos.md delete mode 100644 changelog.d/added-webhook-owner-setup.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6883be0e10..16a3f20cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.240.0) + +### Added + +- **`project.reference_repos`: study-only repos outside the project** ([#18015](https://github.com/hivecommons/hive/issues/18015)). A hive whose deliverable is *about* another project (a docs assessment of `hivecommons/*` landing in `cncf/techdocs`) can list `owner/name` repos it may clone unauthenticated and GET, with the proxy stripping the hive's token and refusing every write by default; the kick tells agents such items are in scope and that findings go in the deliverable in the authorized repo, so the scanner no longer flags them out-of-scope/critical. A rare per-entry `write: [issues, prs]` opt-in allows only opening issues, opening PRs/pushing topic branches and commenting on the hive's own items, using the App's installation on the target org or `write_token_file`; boot refuses write-mode without either (exit 22). Dashboard shows a minimal `reference · <tier>` card. +- Owner-facing GitHub webhook setup: ⚙ Settings → Forge App gains a "GitHub webhooks" section showing the payload URL, events and live receiver status, with **Generate secret** (stored in `/data/github-webhook-secret.key`, shown once) and **Create webhook for me** when the App installation has hooks write permission; a webhooks tile on the Governor card deep-links to it; new owner guide in `docs/webhooks.md` (#18017). + ## 2026-11-12 (v5.239.0) ### Added diff --git a/changelog.d/added-reference-repos.md b/changelog.d/added-reference-repos.md deleted file mode 100644 index 0105e3a710..0000000000 --- a/changelog.d/added-reference-repos.md +++ /dev/null @@ -1 +0,0 @@ -- **`project.reference_repos`: study-only repos outside the project** ([#18015](https://github.com/hivecommons/hive/issues/18015)). A hive whose deliverable is *about* another project (a docs assessment of `hivecommons/*` landing in `cncf/techdocs`) can list `owner/name` repos it may clone unauthenticated and GET, with the proxy stripping the hive's token and refusing every write by default; the kick tells agents such items are in scope and that findings go in the deliverable in the authorized repo, so the scanner no longer flags them out-of-scope/critical. A rare per-entry `write: [issues, prs]` opt-in allows only opening issues, opening PRs/pushing topic branches and commenting on the hive's own items, using the App's installation on the target org or `write_token_file`; boot refuses write-mode without either (exit 22). Dashboard shows a minimal `reference · <tier>` card. diff --git a/changelog.d/added-webhook-owner-setup.md b/changelog.d/added-webhook-owner-setup.md deleted file mode 100644 index 11c7755899..0000000000 --- a/changelog.d/added-webhook-owner-setup.md +++ /dev/null @@ -1 +0,0 @@ -- Owner-facing GitHub webhook setup: ⚙ Settings → Forge App gains a "GitHub webhooks" section showing the payload URL, events and live receiver status, with **Generate secret** (stored in `/data/github-webhook-secret.key`, shown once) and **Create webhook for me** when the App installation has hooks write permission; a webhooks tile on the Governor card deep-links to it; new owner guide in `docs/webhooks.md` (#18017). From 98e8fe2ddb6af7cc7528977ddff7cf8127aa61dc Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 10:23:20 -0500 Subject: [PATCH 131/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20show=20linked?= =?UTF-8?q?=20review=20perspectives=20read-only=20in=20Review=20Gate=20(#1?= =?UTF-8?q?8072)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: show linked review perspectives read-only in Review Gate When review.perspectives_source is set, GET /api/config/review now returns the effective (linked) perspective set instead of the inline fallback, plus perspectives_linked, the source's status/provenance, any resolver warning, and the GitHub blob URL. PUT refuses perspectives/perspective_prompts edits with 409 while linked, before anything is mutated or pushed to the relay; combined_perspectives, plan_match and max_perspectives_per_pr stay editable. The Review Gate Perspectives block renders a "Linked to owner/repo/path@ref" banner with status and warning, disables the checkboxes and focus textareas, and hides "Add perspective", remove and "use built-in". Unlinked hives render exactly as before. Hive-Run: hivecommons/hive#17989 Hive-Plan: hivecommons/hive#17975 live-linked review perspectives Hive-Spec: review-perspectives-source#dashboard-read-only Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🎨 dashboard: move linked-perspectives banner styles into config-dialog.css The Review Gate's linked-source banner added three style= attributes, a raw 0.74rem font-size and a raw 8px 10px padding to the operator index.html, pushing TestStyleRatchet over its baseline. The banner, status and warning lines now use .rv-perspectives-linked* classes built from tokens (--sp-*, --fs-sm, --r, --muted, --yellow), so the counts return to baseline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🐛 github: pin clock in TestReopenedClaimPersistsAcrossLoad The fixture's merge date (2026-10-14) aged past settledClaimRetention (30 days) on 2026-11-13, so LoadClaimLedger pruned the claim on reload using the wall clock and the test began failing on every branch. Drive both the ledger and the reload from a fixed clock near the fixture. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...ded-17989-dashboard-linked-perspectives.md | 1 + src/pkg/dashboard/api_config_review.go | 83 +++++++++++- src/pkg/dashboard/api_config_sections_test.go | 126 ++++++++++++++++++ .../static-src/css/config-dialog.css | 4 + .../static-src/js/config-features.js | 29 +++- src/pkg/dashboard/static/index.html | 33 ++++- src/pkg/github/prclaims_reopen_test.go | 7 +- 7 files changed, 267 insertions(+), 16 deletions(-) create mode 100644 changelog.d/added-17989-dashboard-linked-perspectives.md diff --git a/changelog.d/added-17989-dashboard-linked-perspectives.md b/changelog.d/added-17989-dashboard-linked-perspectives.md new file mode 100644 index 0000000000..37eacda399 --- /dev/null +++ b/changelog.d/added-17989-dashboard-linked-perspectives.md @@ -0,0 +1 @@ +- **Review Gate shows linked review perspectives read-only.** When `review.perspectives_source` links the perspective set to a repo file, Settings → Features → Review Gate → Perspectives now shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line pointing at the file on GitHub, and the resolver's status and any fetch or malformed-document warning. The checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next reload would silently replace; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. Unlinked hives see no change ([#17989](https://github.com/hivecommons/hive/issues/17989)). diff --git a/src/pkg/dashboard/api_config_review.go b/src/pkg/dashboard/api_config_review.go index fb129ebb6e..3a4306da90 100644 --- a/src/pkg/dashboard/api_config_review.go +++ b/src/pkg/dashboard/api_config_review.go @@ -2,6 +2,7 @@ package dashboard import ( "net/http" + "net/url" "sort" "strings" "time" @@ -30,8 +31,32 @@ type reviewSection struct { // section can offer a pick-list for review.reviewer_agents and warn when a // fan-out would run every review on one engine (#13460). ReviewerCandidates []reviewerCandidate `json:"reviewer_candidates"` + // LinkedPerspectiveNames and LinkedPerspectivePrompts shadow the embedded + // review.perspectives / review.perspective_prompts with the EFFECTIVE set + // (ReviewConfig.EffectivePerspectives), so a hive whose perspectives are + // linked to a repo file shows what the reviewer actually runs, not the + // inline fallback the linked set overrides (hivecommons/hive#17989). + LinkedPerspectiveNames []string `json:"perspectives,omitempty"` + LinkedPerspectivePrompts map[string]string `json:"perspective_prompts,omitempty"` + // PerspectivesLinked is true while review.perspectives_source is set. The + // section is then read-only: PUT refuses perspective edits because the + // next reload would replace them with the linked set. + PerspectivesLinked bool `json:"perspectives_linked"` + // PerspectivesSourceStatus is the resolver's provenance for the linked + // set (github:owner/repo@ref, github-cache:owner/repo), or "unresolved" + // when the source is set but nothing has resolved yet, in which case the + // inline perspectives are still in effect. Empty when not linked. + PerspectivesSourceStatus string `json:"perspectives_source_status,omitempty"` + // PerspectivesSourceWarning is the last fetch or malformed-document + // failure while an older set stays in effect. + PerspectivesSourceWarning string `json:"perspectives_source_warning,omitempty"` + // PerspectivesSourceURL is the GitHub blob URL of the linked document. + PerspectivesSourceURL string `json:"perspectives_source_url,omitempty"` } +// perspectivesSourceStatusUnresolved marks a set source with no resolved set. +const perspectivesSourceStatusUnresolved = "unresolved" + // reviewerCandidate is one row of reviewSection.ReviewerCandidates. type reviewerCandidate struct { Name string `json:"name"` @@ -83,7 +108,52 @@ func reviewSectionResponse(cfg *config.Config) reviewSection { if cfg != nil { rc = cfg.Review } - return reviewSection{rc, builtin, reviewerCandidates(cfg)} + out := reviewSection{ReviewConfig: rc, BuiltinPerspectives: builtin, ReviewerCandidates: reviewerCandidates(cfg)} + out.LinkedPerspectiveNames, out.LinkedPerspectivePrompts = rc.EffectivePerspectives() + if src := rc.PerspectivesSource; src.IsSet() { + out.PerspectivesLinked = true + out.PerspectivesSourceURL = perspectivesSourceURL(*src) + out.PerspectivesSourceStatus = perspectivesSourceStatusUnresolved + if lp := rc.LinkedPerspectives; lp != nil { + if lp.Source != "" { + out.PerspectivesSourceStatus = lp.Source + } + out.PerspectivesSourceWarning = lp.Warning + } + } + return out +} + +// perspectivesSourceURL is the github.com blob URL for a linked perspectives +// document. An empty ref resolves to the repo's default branch, which +// GitHub's HEAD alias names without a second API call. Each segment is +// escaped on its own so a slashed ref (release/v5) or nested path keeps its +// separators. +func perspectivesSourceURL(src config.DefinitionSourceConfig) string { + ref := src.Ref + if ref == "" { + ref = "HEAD" + } + return "https://github.com/" + url.PathEscape(src.Owner) + "/" + url.PathEscape(src.Repo) + + "/blob/" + escapePathSegments(ref) + "/" + escapePathSegments(strings.TrimLeft(src.Path, "/")) +} + +func escapePathSegments(p string) string { + parts := strings.Split(p, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +// perspectivesSourceLabel names a linked source as owner/repo/path[@ref] for +// operator-facing messages. +func perspectivesSourceLabel(src config.DefinitionSourceConfig) string { + label := src.Owner + "/" + src.Repo + "/" + strings.TrimLeft(src.Path, "/") + if src.Ref != "" { + label += "@" + src.Ref + } + return label } // handleReviewConfigPut updates the review-swarm merge-gate settings from the @@ -205,6 +275,17 @@ func (s *Server) handleReviewConfigPut(w http.ResponseWriter, r *http.Request) { return } + // A linked perspective set is owned by its repo file: an inline edit + // would be saved and then silently replaced by the next reload, so it is + // refused before anything is mutated. The operator-owned review knobs + // (combined_perspectives, plan_match, max_perspectives_per_pr) stay + // editable (hivecommons/hive#17989). + if src := s.deps.Config.Review.PerspectivesSource; src.IsSet() && (body.Perspectives != nil || body.PerspectivePrompts != nil) { + jsonError(w, "review perspectives are linked to "+perspectivesSourceLabel(*src)+ + "; change them in that file, or remove review.perspectives_source to edit them here", http.StatusConflict) + return + } + if body.MaxParallelReviews != nil && *body.MaxParallelReviews < 0 { jsonError(w, "max_parallel_reviews must be >= 0", http.StatusBadRequest) return diff --git a/src/pkg/dashboard/api_config_sections_test.go b/src/pkg/dashboard/api_config_sections_test.go index f73813f5fd..ea78d6c28e 100644 --- a/src/pkg/dashboard/api_config_sections_test.go +++ b/src/pkg/dashboard/api_config_sections_test.go @@ -798,3 +798,129 @@ func TestReviewConfigGet_ReviewerCandidates(t *testing.T) { } } } + +// A hive whose perspectives are linked to a repo file shows the linked set, +// its source and its provenance, never the inline fallback it overrides +// (hivecommons/hive#17989). +func TestReviewConfigGetShowsLinkedPerspectives(t *testing.T) { + s := covApiServer(t) + s.deps.Config.Review.Perspectives = []string{"correctness"} + s.deps.Config.Review.PerspectivePrompts = map[string]string{"correctness": "inline focus"} + s.deps.Config.Review.PerspectivesSource = &config.DefinitionSourceConfig{ + Owner: "acme", Repo: "policy", Path: "review/perspectives.yaml", Ref: "release/v5", + } + s.deps.Config.Review.LinkedPerspectives = &config.LinkedPerspectives{ + Names: []string{"security", "api-compat"}, + Focus: map[string]string{"api-compat": "public API breakage"}, + Source: "github-cache:acme/policy", + Warning: "review perspectives document acme/policy/review/perspectives.yaml is invalid: bad", + } + + rec := doGetNoRole(s, "/api/config/review") + if rec.Code != http.StatusOK { + t.Fatalf("GET review: expected 200, got %d", rec.Code) + } + var got struct { + Perspectives []string `json:"perspectives"` + PerspectivePrompts map[string]string `json:"perspective_prompts"` + Linked bool `json:"perspectives_linked"` + Source config.DefinitionSourceConfig `json:"perspectives_source"` + Status string `json:"perspectives_source_status"` + Warning string `json:"perspectives_source_warning"` + URL string `json:"perspectives_source_url"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if !got.Linked { + t.Fatal("perspectives_linked = false with a source set") + } + if got.Source.Owner != "acme" || got.Source.Repo != "policy" || got.Source.Path != "review/perspectives.yaml" || got.Source.Ref != "release/v5" { + t.Fatalf("perspectives_source = %+v", got.Source) + } + if strings.Join(got.Perspectives, ",") != "security,api-compat" { + t.Fatalf("perspectives = %v, want the linked set", got.Perspectives) + } + if got.PerspectivePrompts["api-compat"] != "public API breakage" || got.PerspectivePrompts["correctness"] != "" { + t.Fatalf("perspective_prompts = %v, want the linked focus", got.PerspectivePrompts) + } + if got.Status != "github-cache:acme/policy" || !strings.Contains(got.Warning, "is invalid") { + t.Fatalf("status = %q warning = %q", got.Status, got.Warning) + } + if got.URL != "https://github.com/acme/policy/blob/release/v5/review/perspectives.yaml" { + t.Fatalf("perspectives_source_url = %q", got.URL) + } + + // Set but not yet resolved: still linked (and so read-only), but the + // inline set is what runs and the status says so. + s.deps.Config.Review.LinkedPerspectives = nil + got.Perspectives, got.Status, got.Warning = nil, "", "" + if err := json.Unmarshal(doGetNoRole(s, "/api/config/review").Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if !got.Linked || got.Status != perspectivesSourceStatusUnresolved || strings.Join(got.Perspectives, ",") != "correctness" { + t.Fatalf("unresolved: linked=%v status=%q perspectives=%v", got.Linked, got.Status, got.Perspectives) + } + + // Unlinked: the inline set, and no linked metadata at all. + s.deps.Config.Review.PerspectivesSource = nil + var plain map[string]any + if err := json.Unmarshal(doGetNoRole(s, "/api/config/review").Body.Bytes(), &plain); err != nil { + t.Fatalf("decode: %v", err) + } + if plain["perspectives_linked"] != false { + t.Fatalf("unlinked perspectives_linked = %v", plain["perspectives_linked"]) + } + for _, k := range []string{"perspectives_source", "perspectives_source_status", "perspectives_source_warning", "perspectives_source_url"} { + if _, ok := plain[k]; ok { + t.Fatalf("unlinked response carries %s", k) + } + } + if ps, _ := plain["perspectives"].([]any); len(ps) != 1 || ps[0] != "correctness" { + t.Fatalf("unlinked perspectives = %v", plain["perspectives"]) + } +} + +// While linked, a perspective edit is refused with 409 and nothing changes or +// is pushed to the relay; the operator-owned review knobs stay editable. +func TestReviewConfigPutRejectsPerspectiveEditWhileLinked(t *testing.T) { + s := covApiServer(t) + applied := 0 + s.deps.ReviewConfigApplied = func(config.ReviewConfig) { applied++ } + s.deps.Config.Review.Perspectives = []string{"correctness"} + s.deps.Config.Review.PerspectivesSource = &config.DefinitionSourceConfig{ + Owner: "acme", Repo: "policy", Path: "review/perspectives.yaml", + } + + for _, body := range []map[string]any{ + {"perspectives": []string{"security"}}, + {"perspective_prompts": map[string]string{"security": "x"}}, + {"perspectives": []string{"security"}, "combined_perspectives": true}, + } { + rec := doPut(s, "/api/config/review", body) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "acme/policy/review/perspectives.yaml") { + t.Fatalf("%v: expected 409 naming the source, got %d: %s", body, rec.Code, rec.Body.String()) + } + } + rv := s.deps.Config.Review + if strings.Join(rv.Perspectives, ",") != "correctness" || rv.PerspectivePrompts != nil || rv.CombinedPerspectives { + t.Fatalf("refused PUT changed config: %+v", rv) + } + if applied != 0 { + t.Fatalf("ReviewConfigApplied called %d times for a refused PUT", applied) + } + + rec := doPut(s, "/api/config/review", map[string]any{"combined_perspectives": true, "max_perspectives_per_pr": 2}) + if rec.Code != http.StatusOK { + t.Fatalf("combined_perspectives while linked: expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + if !s.deps.Config.Review.CombinedPerspectives || s.deps.Config.Review.MaxPerspectivesPerPR != 2 || applied != 1 { + t.Fatalf("operator-owned knobs not applied: %+v applied=%d", s.deps.Config.Review, applied) + } + + // A partially filled source is unset, so perspectives stay editable. + s.deps.Config.Review.PerspectivesSource = &config.DefinitionSourceConfig{Owner: "acme", Repo: "policy"} + if rec := doPut(s, "/api/config/review", map[string]any{"perspectives": []string{"security"}}); rec.Code != http.StatusOK { + t.Fatalf("partial source: expected 200, got %d: %s", rec.Code, rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/static-src/css/config-dialog.css b/src/pkg/dashboard/static-src/css/config-dialog.css index 52c22bfa41..8dc476ea89 100644 --- a/src/pkg/dashboard/static-src/css/config-dialog.css +++ b/src/pkg/dashboard/static-src/css/config-dialog.css @@ -263,6 +263,10 @@ .rv-reviewer-help { margin: var(--sp-1) var(--sp-0) var(--sp-0); } .rv-reviewer-help-above { margin: var(--sp-0) var(--sp-0) var(--sp-2); } #rv-require-labels-field[hidden] { display: none; } + /* Linked review.perspectives_source banner in the Review Gate (#17989). */ + .rv-perspectives-linked { margin: var(--sp-0) var(--sp-0) var(--sp-3); padding: var(--sp-4) var(--sp-5); background: var(--surface); border: 1px solid var(--border); border-radius: var(--r); font-size: var(--fs-sm); } + .rv-perspectives-linked-status, .rv-perspectives-linked-warning { margin-top: var(--sp-2); color: var(--muted); } + .rv-perspectives-linked-warning { color: var(--yellow); } .am-trusted-authors-repos { min-height: 72px; } .am-bot-empty { color: var(--muted); margin: var(--sp-1) 0; } .am-bot-add { display: flex; gap: var(--sp-3); margin-top: var(--sp-4); align-items: center; } diff --git a/src/pkg/dashboard/static-src/js/config-features.js b/src/pkg/dashboard/static-src/js/config-features.js index 4677f288de..82a2912536 100644 --- a/src/pkg/dashboard/static-src/js/config-features.js +++ b/src/pkg/dashboard/static-src/js/config-features.js @@ -226,17 +226,31 @@ .concat(rvSelected.filter(n => !rvBuiltinNames.includes(n))) .filter((n, i, a) => a.indexOf(n) === i); const rvIsOn = (name) => rvSelected.length === 0 ? rvBuiltinNames.includes(name) || rvCustomNames.includes(name) : rvSelected.includes(name); + // Linked to a repo file (review.perspectives_source): the set shown is + // the linked one and every control is read-only, because the API + // refuses inline edits the next reload would replace (#17989). + const rvLinked = rv.perspectives_linked === true; + const rvSrc = rv.perspectives_source || {}; + const rvSrcLabel = `${rvSrc.owner || ''}/${rvSrc.repo || ''}/${rvSrc.path || ''}${rvSrc.ref ? '@' + rvSrc.ref : ''}`; + const rvSrcStatus = rv.perspectives_source_status || ''; + const rvSrcURL = /^https:\/\/github\.com\//.test(rv.perspectives_source_url || '') ? rv.perspectives_source_url : ''; + const rvLinkedBanner = !rvLinked ? '' : `<div class="rv-perspectives-linked"> + <div>Linked to ${rvSrcURL ? `<a href="${esc(rvSrcURL)}" target="_blank" rel="noopener noreferrer"><code>${esc(rvSrcLabel)}</code> ↗</a>` : `<code>${esc(rvSrcLabel)}</code>`} ${yamlHint('review.perspectives_source')} <span class="text-muted">— read-only here; change the perspectives in that file, or remove the link from hive.yaml to edit them inline.</span></div> + <div class="rv-perspectives-linked-status">${rvSrcStatus === 'unresolved' ? 'Not resolved yet (repo unreachable, not allowlisted, or invalid) — the inline perspectives below are in effect.' : `Status: <code>${esc(rvSrcStatus)}</code>`}</div> + ${rv.perspectives_source_warning ? `<div class="rv-perspectives-linked-warning">⚠ ${esc(rv.perspectives_source_warning)}</div>` : ''} + </div>`; + const rvLockAttr = rvLinked ? ' disabled' : ''; const rvPerspectiveRow = (name, defaultFocus) => { const custom = rvPrompts[name] || ''; const isCustom = !rvBuiltinNames.includes(name); return `<div class="rv-perspective" data-perspective="${esc(name)}" style="margin-top:var(--sp-4);padding:8px 10px;border:1px solid var(--border);border-radius:var(--r)"> - <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:pointer;font-weight:600;font-size:0.8rem"> - <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''} data-change-action="gh91"> + <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:${rvLinked ? 'default' : 'pointer'};font-weight:600;font-size:0.8rem"> + <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''}${rvLockAttr} data-change-action="gh91"> <code>${esc(name)}</code>${isCustom ? ' <span style="font-size:var(--fs-xs);color:var(--muted);font-weight:400">custom</span>' : ''} - ${isCustom ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} + ${isCustom && !rvLinked ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} </label> - <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}" data-change-action="gh92">${esc(custom)}</textarea> - ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} + <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}"${rvLockAttr} data-change-action="gh92">${esc(custom)}</textarea> + ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom && !rvLinked ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} </div>`; }; const rvRec = rv.recommendations || {}; @@ -486,13 +500,14 @@ </div> <div id="rv-perspectives" style="margin-top:10px"> <p style="font-size:0.72rem;color:var(--muted);margin:var(--sp-0) var(--sp-0) var(--sp-2)">Which questions the reviewer asks of every PR, and what each one looks for. Edit the text to tell this hive's reviewer what matters in <em>your</em> repos — e.g. name the conventions doc for <code>style</code>. Blank means the built-in. Unchecking a perspective stops it running and stops its verdicts being accepted.</p> + ${rvLinkedBanner} ${rvBuiltin.map(b => rvPerspectiveRow(b.name, b.focus)).join('')} ${rvCustomNames.map(n => rvPerspectiveRow(n, '')).join('')} - <div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> + ${rvLinked ? '' : `<div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> <input type="text" id="rv-new-perspective" placeholder="new perspective, e.g. api-compatibility" style="flex:1;font-size:0.76rem" pattern="[a-z][a-z0-9]*(-[a-z0-9]+)*"> <button type="button" class="hv-btn btn-secondary btn-sm btn-small" data-action="gh95">Add perspective</button> </div> - <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p> + <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p>`} </div> <div class="config-subgroup">Reviewers</div> <div class="config-field" id="rv-reviewer-agents"> diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 065d33f36e..e869b6c771 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -4728,6 +4728,10 @@ .rv-reviewer-help { margin: var(--sp-1) var(--sp-0) var(--sp-0); } .rv-reviewer-help-above { margin: var(--sp-0) var(--sp-0) var(--sp-2); } #rv-require-labels-field[hidden] { display: none; } + /* Linked review.perspectives_source banner in the Review Gate (#17989). */ + .rv-perspectives-linked { margin: var(--sp-0) var(--sp-0) var(--sp-3); padding: var(--sp-4) var(--sp-5); background: var(--surface); border: 1px solid var(--border); border-radius: var(--r); font-size: var(--fs-sm); } + .rv-perspectives-linked-status, .rv-perspectives-linked-warning { margin-top: var(--sp-2); color: var(--muted); } + .rv-perspectives-linked-warning { color: var(--yellow); } .am-trusted-authors-repos { min-height: 72px; } .am-bot-empty { color: var(--muted); margin: var(--sp-1) 0; } .am-bot-add { display: flex; gap: var(--sp-3); margin-top: var(--sp-4); align-items: center; } @@ -68580,17 +68584,31 @@ <h3 class="features-section-title">${esc(s.title)}</h3> .concat(rvSelected.filter(n => !rvBuiltinNames.includes(n))) .filter((n, i, a) => a.indexOf(n) === i); const rvIsOn = (name) => rvSelected.length === 0 ? rvBuiltinNames.includes(name) || rvCustomNames.includes(name) : rvSelected.includes(name); + // Linked to a repo file (review.perspectives_source): the set shown is + // the linked one and every control is read-only, because the API + // refuses inline edits the next reload would replace (#17989). + const rvLinked = rv.perspectives_linked === true; + const rvSrc = rv.perspectives_source || {}; + const rvSrcLabel = `${rvSrc.owner || ''}/${rvSrc.repo || ''}/${rvSrc.path || ''}${rvSrc.ref ? '@' + rvSrc.ref : ''}`; + const rvSrcStatus = rv.perspectives_source_status || ''; + const rvSrcURL = /^https:\/\/github\.com\//.test(rv.perspectives_source_url || '') ? rv.perspectives_source_url : ''; + const rvLinkedBanner = !rvLinked ? '' : `<div class="rv-perspectives-linked"> + <div>Linked to ${rvSrcURL ? `<a href="${esc(rvSrcURL)}" target="_blank" rel="noopener noreferrer"><code>${esc(rvSrcLabel)}</code> ↗</a>` : `<code>${esc(rvSrcLabel)}</code>`} ${yamlHint('review.perspectives_source')} <span class="text-muted">— read-only here; change the perspectives in that file, or remove the link from hive.yaml to edit them inline.</span></div> + <div class="rv-perspectives-linked-status">${rvSrcStatus === 'unresolved' ? 'Not resolved yet (repo unreachable, not allowlisted, or invalid) — the inline perspectives below are in effect.' : `Status: <code>${esc(rvSrcStatus)}</code>`}</div> + ${rv.perspectives_source_warning ? `<div class="rv-perspectives-linked-warning">⚠ ${esc(rv.perspectives_source_warning)}</div>` : ''} + </div>`; + const rvLockAttr = rvLinked ? ' disabled' : ''; const rvPerspectiveRow = (name, defaultFocus) => { const custom = rvPrompts[name] || ''; const isCustom = !rvBuiltinNames.includes(name); return `<div class="rv-perspective" data-perspective="${esc(name)}" style="margin-top:var(--sp-4);padding:8px 10px;border:1px solid var(--border);border-radius:var(--r)"> - <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:pointer;font-weight:600;font-size:0.8rem"> - <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''} data-change-action="gh91"> + <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:${rvLinked ? 'default' : 'pointer'};font-weight:600;font-size:0.8rem"> + <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''}${rvLockAttr} data-change-action="gh91"> <code>${esc(name)}</code>${isCustom ? ' <span style="font-size:var(--fs-xs);color:var(--muted);font-weight:400">custom</span>' : ''} - ${isCustom ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} + ${isCustom && !rvLinked ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} </label> - <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}" data-change-action="gh92">${esc(custom)}</textarea> - ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} + <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}"${rvLockAttr} data-change-action="gh92">${esc(custom)}</textarea> + ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom && !rvLinked ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} </div>`; }; const rvRec = rv.recommendations || {}; @@ -68840,13 +68858,14 @@ <h3 class="features-section-title">${esc(s.title)}</h3> </div> <div id="rv-perspectives" style="margin-top:10px"> <p style="font-size:0.72rem;color:var(--muted);margin:var(--sp-0) var(--sp-0) var(--sp-2)">Which questions the reviewer asks of every PR, and what each one looks for. Edit the text to tell this hive's reviewer what matters in <em>your</em> repos — e.g. name the conventions doc for <code>style</code>. Blank means the built-in. Unchecking a perspective stops it running and stops its verdicts being accepted.</p> + ${rvLinkedBanner} ${rvBuiltin.map(b => rvPerspectiveRow(b.name, b.focus)).join('')} ${rvCustomNames.map(n => rvPerspectiveRow(n, '')).join('')} - <div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> + ${rvLinked ? '' : `<div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> <input type="text" id="rv-new-perspective" placeholder="new perspective, e.g. api-compatibility" style="flex:1;font-size:0.76rem" pattern="[a-z][a-z0-9]*(-[a-z0-9]+)*"> <button type="button" class="hv-btn btn-secondary btn-sm btn-small" data-action="gh95">Add perspective</button> </div> - <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p> + <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p>`} </div> <div class="config-subgroup">Reviewers</div> <div class="config-field" id="rv-reviewer-agents"> diff --git a/src/pkg/github/prclaims_reopen_test.go b/src/pkg/github/prclaims_reopen_test.go index d95255b93e..651cfc234c 100644 --- a/src/pkg/github/prclaims_reopen_test.go +++ b/src/pkg/github/prclaims_reopen_test.go @@ -233,8 +233,13 @@ func TestReconcileReopenedIssuesVoidsClaimFromTimeline(t *testing.T) { } func TestReopenedClaimPersistsAcrossLoad(t *testing.T) { + // Pin the clock near the fixture dates: with the wall clock, the merged + // claim ages past settledClaimRetention 30 days after reopenTestMerged and + // the reload prunes it, which is retention working, not a reopen bug. + now := func() time.Time { return reopenTestReopened.Add(time.Hour) } path := filepath.Join(t.TempDir(), "ledger.json") ledger := NewClaimLedger(path, testLogger()) + ledger.SetClock(now) ledger.Reconcile([]IssueClaim{reopenTestClaim()}, true) if !ledger.MarkIssueReopened("r", 7, reopenTestReopened) { t.Fatal("setup: reopen not recorded") @@ -242,7 +247,7 @@ func TestReopenedClaimPersistsAcrossLoad(t *testing.T) { if err := ledger.Save(); err != nil { t.Fatal(err) } - reloaded, err := LoadClaimLedger(path, testLogger()) + reloaded, err := loadClaimLedgerAt(path, testLogger(), now) if err != nil { t.Fatal(err) } From 8db21ec69032241ea5b38af37c84ea71332288c8 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 10:33:23 -0500 Subject: [PATCH 132/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20project.referenc?= =?UTF-8?q?e=5Frepos=20in=20agent-configuration=20and=20hive.yaml.example?= =?UTF-8?q?=20(#18073)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: project.reference_repos in agent-configuration and hive.yaml.example The operator reference and github-write-surface already document reference_repos (merged in #18023). Add the agent-facing explanation — config shape, read-only default, the exact opt-in write surface and what it still refuses, the credential rule, kick and dashboard rendering — to agent-configuration.md, and a commented example to hive.yaml.example. Hive-Run: hivecommons/hive#18022 Hive-Plan: #18015 child 5 of 5 (docs) Hive-Spec: project.reference_repos#docs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * changelog: fragment for #18073 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-18073.md | 1 + src/docs/agent-configuration.md | 26 ++++++++++++++++++++++++++ src/hive.yaml.example | 16 ++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 changelog.d/changed-18073.md diff --git a/changelog.d/changed-18073.md b/changelog.d/changed-18073.md new file mode 100644 index 0000000000..82b08a446e --- /dev/null +++ b/changelog.d/changed-18073.md @@ -0,0 +1 @@ +- docs: project.reference_repos in agent-configuration and hive.yaml.example (#18073) diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 49a8f6bba0..fbff124ed5 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1157,6 +1157,32 @@ project: agent_issue_decision_policy: resolve # resolve (default) or escalate ``` +### Reference repos: what agents may study (`project.reference_repos`) + +Some hives produce a deliverable *about* another project — a CNCF TechDocs hive whose authorized repo is `cncf/techdocs` assesses `hivecommons/docs`, for example. Without configuration, the scanner reads such an item as out-of-scope or critical, and the proxy has no way to say "read this repo but do not work it". `project.reference_repos` names those repos ([#18015](https://github.com/hivecommons/hive/issues/18015)). A reference repo is never an authorized repo: it gets no queue, no ACMM evaluation and no work items, and it cannot also be in `project.repos`. + +Each entry is either a string (`owner/name`, read-only) or a mapping `{repo, write, write_token_file}`. Bare names are rejected because a reference repo lives outside the project org. + +```yaml +project: + repos: [techdocs] + reference_repos: + - hivecommons/docs # read-only (default) + - hivecommons/hivecommons.github.io + - repo: hivecommons/hive + write: [issues, prs] # opt-in tier + # write_token_file: /data/secrets/hive-ref-pat # only if the App is not installed on that org +``` + +| Tier | Agents may | Proxy refuses | +|---|---|---| +| `read-only` (string form, or no `write`) | `git clone`/fetch over HTTPS, GitHub REST/GraphQL GETs. Every request goes out without auth because the hive's App token is removed, so a private reference repo is not readable. | Every non-GET, with a 403 that names `reference_repos`. | +| `issues`, `prs` or `issues+prs` (`write:`) | The reads above, plus only: open an issue (`issues`), open a PR or push a topic branch (`prs`), and comment on an issue or PR the hive itself opened. | Merges, pushes to the default branch, labels, assignees, reviews, and closing other people's items. Each refusal is logged as `reference_repo` in the violations log, so retrying cannot succeed. | + +Writes on the opt-in tier use a credential valid for *that* repo, never the hive's own installation token. By default the same GitHub App must be installed on the target org: the hive discovers that installation and mints a token limited to the one repo and the opted-in permissions. Otherwise, set `write_token_file`. If a write-mode entry has neither, boot fails with exit code 22. These are direct POSTs. The `hive-open-issue`/`hive-open-pr` relays only act on `project.repos`, so a lane under `write_surface.enforce` cannot write to a reference repo at all ([GitHub write surface](github-write-surface.md#reference-repos)). + +What agents see: every kick carries a `📚 REFERENCE REPOS` block listing each repo with its tier (`[reference · read-only]`, `[reference · issues+prs]`, …). It tells agents that an authorized-repo item whose deliverables name a reference repo is **in scope**. Findings belong in the deliverable inside the authorized repo, unless the tier allows an issue or PR. On the dashboard, each reference repo is a minimal card with a `reference · <tier>` chip and no stats, ACMM level or work items. Reference repos come in a separate payload field, so no queue or ACMM consumer counts them. Field-by-field defaults are in the [operator reference](operator-reference/README.md#notable-fields). + ## Label policy: which issues agents may work There is exactly **one** label-policy surface for the hive's own agents — the **Governor Configuration → Labels** tab — and it has two polarities: diff --git a/src/hive.yaml.example b/src/hive.yaml.example index cbd09db9a6..2dcfdf7cf7 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -39,6 +39,22 @@ project: # - repo: repo-two # by: your-github-login # reason: release freeze until Friday + # Optional: repos OUTSIDE the project that agents may study but not work + # (src/docs/agent-configuration.md#reference-repos-what-agents-may-study-projectreference_repos). + # Always owner/name; never also in `repos`. No queue, ACMM or work items. + # String form = read-only: clone/fetch and API GETs, sent unauthenticated + # (the hive token is stripped, so private repos are unreadable); every + # non-GET is refused with a 403 naming reference_repos. `write` opts in to + # ONLY: open an issue, open a PR / push a topic branch, comment on the hive's + # own issue/PR — never merge, default-branch push, labels, assignees, reviews + # or closing others' items. Writes use the App installed on that org (minted + # for the one repo) or write_token_file; boot exits 22 with neither. Kicks + # list them as 📚 REFERENCE REPOS; the dashboard shows a `reference · <tier>` chip. + # reference_repos: + # - hivecommons/docs # read-only + # - repo: hivecommons/hive + # write: [issues, prs] # issues, prs, or both + # write_token_file: /data/secrets/ref-pat # optional; only with write # Optional opt-in "clanker-requested" policy, under project.issue_filter.reporter_trust. # Off by default. While on, a PR carrying the label below is treated as held. From 60ed067c58fe2a71146a372eeb7ee8efa0596f09 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 10:53:07 -0500 Subject: [PATCH 133/223] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20webhooks=20?= =?UTF-8?q?card=20names=20the=20App=20hook=20permission;=20pin=20card=20co?= =?UTF-8?q?py=20in=20tests=20(#18077)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Forge App → GitHub webhooks (#18017, shipped in #18024) hid the "Create webhook for me" option without explanation when the App installation lacks organization_hooks / repository_hooks write. Render a one-line hint naming the permission to grant instead. Tests: node-executed render tests for the relay, none and env-secret verification modes (relay copy vs set-up steps, generate-secret button, create button vs permission hint), and a Settings parity test that every endpoint the card fetches is in the page, routed with that method and owner-only. Hive-Run: hivecommons/hive#18029 Hive-Plan: #18017 child 4 of 5 (dashboard webhooks card) Hive-Spec: github-webhooks-owner-setup#dashboard-card Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...ged-18029-webhooks-card-permission-hint.md | 1 + .../dashboard/config_webhooks_card_js_test.go | 89 +++++++++++++++++++ .../dashboard/github_settings_parity_test.go | 23 +++++ .../static-src/js/config-webhooks.js | 4 +- src/pkg/dashboard/static/index.html | 4 +- 5 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 changelog.d/changed-18029-webhooks-card-permission-hint.md create mode 100644 src/pkg/dashboard/config_webhooks_card_js_test.go diff --git a/changelog.d/changed-18029-webhooks-card-permission-hint.md b/changelog.d/changed-18029-webhooks-card-permission-hint.md new file mode 100644 index 0000000000..8b89d7cf4a --- /dev/null +++ b/changelog.d/changed-18029-webhooks-card-permission-hint.md @@ -0,0 +1 @@ +- **Settings → Forge App → GitHub webhooks names the App permission to grant** (#18029): when the GitHub App installation cannot manage hooks, the card now says which permission to grant (Organization → Webhooks `organization_hooks`, or Repository → Webhooks `repository_hooks`, Read and write) so **Create webhook for me** appears, instead of silently omitting the option. Node-executed tests pin the card's copy for relay, none and env-secret modes, and a parity test checks every endpoint the card calls is routed and owner-only. diff --git a/src/pkg/dashboard/config_webhooks_card_js_test.go b/src/pkg/dashboard/config_webhooks_card_js_test.go new file mode 100644 index 0000000000..64d9b5b5c2 --- /dev/null +++ b/src/pkg/dashboard/config_webhooks_card_js_test.go @@ -0,0 +1,89 @@ +package dashboard + +import ( + "os/exec" + "strings" + "testing" +) + +// Settings → Forge App → GitHub webhooks (hivecommons/hive#18029): the card +// is rendered from GET /api/config/github/webhooks alone. These tests lift the +// render functions out of index.html and execute them under node against the +// three verification modes the endpoint reports, so the copy an owner reads +// for each mode is pinned, not only pattern-matched in the source. + +func webhooksCardHarness(t *testing.T, html string) string { + t.Helper() + return strings.Join([]string{ + "const assert = require('assert');", + "let _webhookSetup = null; let _webhookSecretOnce = ''; let _webhookConfirmRegenerate = false;", + "const host = { innerHTML: '' };", + "global.document = { getElementById(id) { return id === 'webhook-setup-host' ? host : null; } };", + jsFunc(t, html, "esc"), + jsFunc(t, html, "forgeAppRow"), + jsFunc(t, html, "webhookStatusChip"), + jsFunc(t, html, "webhookAgo"), + jsFunc(t, html, "webhookCopyBtn"), + jsFunc(t, html, "renderWebhookSecretBlock"), + jsFunc(t, html, "renderWebhookCreateBlock"), + jsFunc(t, html, "renderWebhookSetup"), + "function render(d) { _webhookSetup = d; renderWebhookSetup(); return host.innerHTML; }", + `const base = { payload_url: 'https://spoke.example/api/webhook/github', payload_url_source: 'config', + path: '/api/webhook/github', content_type: 'application/json', events: ['check_run', 'pull_request', 'push'], + optional_events: {}, org: 'o', repos: ['r'], app_org_hooks: false, app_repo_hooks: false };`, + }, "\n") +} + +func runWebhooksCardScript(t *testing.T, script string) { + t.Helper() + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not on PATH — the webhooks card render was NOT executed by this run") + } + if out, err := runNodeStdin(node, script); err != nil { + t.Fatalf("node failed: %v\n%s", err, out) + } +} + +func TestWebhooksCardRendersRelayAndSetupCopy(t *testing.T) { + html := indexHTML(t) + runWebhooksCardScript(t, webhooksCardHarness(t, html)+` +// relay: a hosted spoke receives hub-relayed deliveries; nothing to set up, live status shown. +const relay = render(Object.assign({}, base, { verification: 'relay (hub-signed)', hosted: true, + secret_source: '', secret_configured: false, + health: { events_1h: 4, last_event_at: new Date(Date.now() - 120000).toISOString(), mode: 'webhook' } })); +assert.ok(relay.includes('hub relay; the GitHub App installation delivers through the hub, nothing to set up'), relay); +assert.ok(relay.includes('Direct webhook (optional)'), relay); +assert.ok(relay.includes('receiving · 4/1h'), relay); +assert.ok(!relay.includes('Set up a webhook on GitHub'), relay); +assert.ok(!relay.includes('No events yet'), relay); + +// none: the set-up steps — generate a secret, payload URL, content type, events to tick. +const none = render(Object.assign({}, base, { verification: 'none', hosted: false, + secret_source: '', secret_configured: false, health: {} })); +assert.ok(none.includes('Set up a webhook on GitHub'), none); +assert.ok(none.includes('not configured'), none); +assert.ok(none.includes('No secret yet — generate one'), none); +assert.ok(none.includes('data-action="webhookGenerateSecret"') && none.includes('>Generate secret<'), none); +assert.ok(none.includes('https://spoke.example/api/webhook/github'), none); +assert.ok(none.includes('application/json'), none); +for (const e of base.events) assert.ok(none.includes('<code class="wh-code">' + e + '</code>'), e); +assert.ok(!none.includes('hub relay'), none); +// Without the App's hook permission there is no create button, only a hint naming the permission. +assert.ok(!none.includes('Create webhook for me'), none); +assert.ok(none.includes('<code>organization_hooks</code>') && none.includes('<code>repository_hooks</code>'), none); + +// direct, env secret: no generate button, the env var is named; no events yet → test-delivery hint. +const env = render(Object.assign({}, base, { verification: 'github-signed', hosted: false, + secret_source: 'env', secret_configured: true, health: {} })); +assert.ok(env.includes('GITHUB_WEBHOOK_SECRET'), env); +assert.ok(!env.includes('webhookGenerateSecret'), env); +assert.ok(env.includes('No events yet'), env); + +// With the App's hook permission the create button replaces the hint. +const app = render(Object.assign({}, base, { verification: 'github-signed', hosted: false, + secret_source: 'file', secret_configured: true, app_org_hooks: true, create_available: true, health: {} })); +assert.ok(app.includes('Create webhook for me'), app); +assert.ok(!app.includes('<code>organization_hooks</code>'), app); +`) +} diff --git a/src/pkg/dashboard/github_settings_parity_test.go b/src/pkg/dashboard/github_settings_parity_test.go index ac31266870..e7f8b21b41 100644 --- a/src/pkg/dashboard/github_settings_parity_test.go +++ b/src/pkg/dashboard/github_settings_parity_test.go @@ -3,6 +3,7 @@ package dashboard import ( "encoding/json" "net/http" + "net/http/httptest" "path/filepath" "strings" "testing" @@ -259,3 +260,25 @@ func TestParityBatch6SettingsRenderEveryKey(t *testing.T) { t.Error("saveForgeGitHubSettings does not save through PUT /api/config/github") } } + +// Settings → Forge App → GitHub webhooks (hivecommons/hive#18029): every +// endpoint the card calls is in the page and routed with the method the card +// uses, and each is owner-only (403, never 404/405, for a non-owner). +func TestConfigGitHubWebhooksCardEndpointsParity(t *testing.T) { + html := indexHTML(t) + srv := newFullServer(t) + for _, ep := range []struct{ method, path string }{ + {http.MethodGet, "/api/config/github/webhooks"}, + {http.MethodPost, "/api/config/github/webhooks/secret"}, + {http.MethodPost, "/api/config/github/webhooks/create"}, + } { + if !strings.Contains(html, "fetch('"+ep.path+"'") { + t.Errorf("Settings UI does not call %s", ep.path) + } + rec := httptest.NewRecorder() + srv.mux.ServeHTTP(rec, httptest.NewRequest(ep.method, ep.path, nil)) + if rec.Code != http.StatusForbidden { + t.Errorf("non-owner %s %s: %d, want 403", ep.method, ep.path, rec.Code) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-webhooks.js b/src/pkg/dashboard/static-src/js/config-webhooks.js index 986ff42f93..257a1ba44a 100644 --- a/src/pkg/dashboard/static-src/js/config-webhooks.js +++ b/src/pkg/dashboard/static-src/js/config-webhooks.js @@ -143,7 +143,9 @@ if (d.app_probe_error) { return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; } - if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + if (!d.app_org_hooks && !d.app_repo_hooks) { + return `<div class="wh-muted wh-note">To have the App create the hook for you, grant it <b>Organization → Webhooks</b> (<code>organization_hooks</code>) or <b>Repository → Webhooks</b> (<code>repository_hooks</code>) at <b>Read and write</b>, then reopen this tab.</div>`; + } const scopes = []; if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index e869b6c771..342530dd61 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -59636,7 +59636,9 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> if (d.app_probe_error) { return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; } - if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + if (!d.app_org_hooks && !d.app_repo_hooks) { + return `<div class="wh-muted wh-note">To have the App create the hook for you, grant it <b>Organization → Webhooks</b> (<code>organization_hooks</code>) or <b>Repository → Webhooks</b> (<code>repository_hooks</code>) at <b>Read and write</b>, then reopen this tab.</div>`; + } const scopes = []; if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); From f33758f5ad4ac50a458a7c4a2120547126f5b039 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:59:09 +0000 Subject: [PATCH 134/223] [scanner] fix: de-flake TestMarketplaceRefreshRoutesAndStatus and TestBackfillCoveredByLedger_HealsRecordBelow040 (#18079) Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- src/pkg/github/stage_ledger_backfill_test.go | 16 ++++++++++++---- src/pkg/hub/marketplace_test.go | 7 ++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/pkg/github/stage_ledger_backfill_test.go b/src/pkg/github/stage_ledger_backfill_test.go index 70d182408a..d93aa802cd 100644 --- a/src/pkg/github/stage_ledger_backfill_test.go +++ b/src/pkg/github/stage_ledger_backfill_test.go @@ -177,14 +177,22 @@ func TestBackfillCoveredByLedger_HealsRecordBelow040(t *testing.T) { c.SetIssueStageLabels(func() IssueStagePolicy { return IssueStagePolicy{Enabled: true} }) store := stagelabels.Shared() issue := c.IssueStageRef("o/r", 7) - store.Observe(issue.Key(), stagelabels.Actionable, "actionable") covered := c.IssueStageRef("o/r", 9) - store.Observe(covered.Key(), stagelabels.CoveredByPR, "open PR covers it") - t.Cleanup(func() { + unseen := c.IssueStageRef("o/r", 11) + // The store is process-wide and other tests in this package drive stage + // facts for o/r#9 / o/r#11 without forgetting them; under -shuffle a + // leaked sub-040 record for the "never seen" #11 made the backfill count + // it too (checked=2). Start from a clean slate for every key we assert on. + forget := func() { store.Forget(issue.Key()) store.Forget(covered.Key()) + store.Forget(unseen.Key()) store.Forget(c.PRStageRef("o/r", 8).Key()) - }) + } + forget() + t.Cleanup(forget) + store.Observe(issue.Key(), stagelabels.Actionable, "actionable") + store.Observe(covered.Key(), stagelabels.CoveredByPR, "open PR covers it") ledger := backfillLedger(t, IssueClaim{Repo: "r", Issue: 7, PRNumber: 8, PRRepo: "r", Closing: true}, IssueClaim{Repo: "r", Issue: 9, PRNumber: 8, PRRepo: "r", Closing: true}, diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index b364fea6dd..531ef4dc70 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -598,6 +598,11 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { } rf.mu.Lock() rf.running = false + // The single-repo refresh above already left an "on-demand:" result in + // rf.last, and rf.running stays false until the background goroutine + // starts — so waiting on those alone can pass before the full run even + // begins. Wait for the full run to replace this result instead. + previous := rf.last rf.mu.Unlock() t.Setenv(hubMarketplaceExtraReposEnv, "konflux-ci/konflux-ci") if rec = mktDo(s, admin("/api/marketplace/refresh")); rec.Code != 202 { @@ -606,7 +611,7 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { testutil.Eventually(t, 5*time.Second, func() bool { rf.mu.Lock() defer rf.mu.Unlock() - return !rf.running && rf.last != nil && strings.HasPrefix(rf.last.Trigger, "on-demand:") + return !rf.running && rf.last != nil && rf.last != previous && strings.HasPrefix(rf.last.Trigger, "on-demand:") }, "background refresh did not finish") rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status", nil)) From cc73479fe7ba0510acd9dd534a14a60497e90a56 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 16:03:20 +0000 Subject: [PATCH 135/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.241.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.241.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-17989-dashboard-linked-perspectives.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-17989-dashboard-linked-perspectives.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a3f20cf5..03461444e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.241.0) + +### Added + +- **Review Gate shows linked review perspectives read-only.** When `review.perspectives_source` links the perspective set to a repo file, Settings → Features → Review Gate → Perspectives now shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line pointing at the file on GitHub, and the resolver's status and any fetch or malformed-document warning. The checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next reload would silently replace; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. Unlinked hives see no change ([#17989](https://github.com/hivecommons/hive/issues/17989)). + ## 2026-11-13 (v5.240.0) ### Added diff --git a/changelog.d/added-17989-dashboard-linked-perspectives.md b/changelog.d/added-17989-dashboard-linked-perspectives.md deleted file mode 100644 index 37eacda399..0000000000 --- a/changelog.d/added-17989-dashboard-linked-perspectives.md +++ /dev/null @@ -1 +0,0 @@ -- **Review Gate shows linked review perspectives read-only.** When `review.perspectives_source` links the perspective set to a repo file, Settings → Features → Review Gate → Perspectives now shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line pointing at the file on GitHub, and the resolver's status and any fetch or malformed-document warning. The checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next reload would silently replace; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. Unlinked hives see no change ([#17989](https://github.com/hivecommons/hive/issues/17989)). From 3aebc9241e63366b09e403903fc035fdfc34679d Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:13:47 -0500 Subject: [PATCH 136/223] =?UTF-8?q?=F0=9F=90=9B=20benchmarks:=20emit=20can?= =?UTF-8?q?onical=20model=20id=20and=20effort=20alongside=20the=20reported?= =?UTF-8?q?=20id=20(#18037)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 benchmarks: emit canonical model id and effort alongside the reported id Different CLIs report one model under different spellings (claude-sonnet-5-5 vs claude-sonnet-5.5, openai-codex/gpt-6.1-sol vs gpt-6.1-sol, gemini-3.8-flash-low), so the leaderboard showed one model as several rows. modelfamily.Resolve folds a raw id into canonical id, vendor and reasoning effort; the benchmarks exporter publishes model.canonical / model.effort next to the raw model.name and derives model.provider from the model family instead of the CLI prefix. Fixes #18036 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🧪 modelfamily: cover canonical-id edge cases for the coverage gate Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> --- .../fixed-benchmarks-canonical-model-id.md | 1 + src/docs/benchmarks.md | 2 +- src/pkg/evidence/benchmarks/convert.go | 29 ++-- src/pkg/evidence/benchmarks/convert_test.go | 17 ++ src/pkg/evidence/benchmarks/record.go | 12 +- .../benchmarks/testdata/result.schema.json | 6 +- src/pkg/modelfamily/modelfamily.go | 159 +++++++++++++++++- src/pkg/modelfamily/modelfamily_test.go | 49 ++++++ 8 files changed, 243 insertions(+), 32 deletions(-) create mode 100644 changelog.d/fixed-benchmarks-canonical-model-id.md diff --git a/changelog.d/fixed-benchmarks-canonical-model-id.md b/changelog.d/fixed-benchmarks-canonical-model-id.md new file mode 100644 index 0000000000..044a0042fc --- /dev/null +++ b/changelog.d/fixed-benchmarks-canonical-model-id.md @@ -0,0 +1 @@ +- Benchmarks export now adds `model.canonical` and `model.effort` next to the reported id and derives `model.provider` from the model family, so `claude-sonnet-5-5` (Copilot CLI), `claude-sonnet-5.5` (Claude Code) and `openai-codex/gpt-6.1-sol` no longer appear as separate models on hivecommons.dev/benchmarks. diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md index ef2ef3c072..5641dae5fe 100644 --- a/src/docs/benchmarks.md +++ b/src/docs/benchmarks.md @@ -22,7 +22,7 @@ One record per PR per role, conforming to | `source.hive_id` | **Salted hash** of the hive id: `sha256("hivecommons-benchmarks/v1/" + hive_id)[:16]`. The hive id itself never leaves the spoke. | | `source.hive_version`, `source.cluster` | Build version and `HIVE_CLUSTER_ID`. | | `agent.name`, `agent.role` (`author` / `reviewer`), `agent.harness` | Bundle `author.agent`, `author.backend`; reviewer records come from the dominant `(backend, model)` among the swarm verdicts. | -| `model.provider`, `model.name` | Bundle `author.model`; the provider is inferred from the model name prefix. | +| `model.provider`, `model.name`, `model.canonical`, `model.effort` | Bundle `author.model`. `name` is the id exactly as the harness reported it; `canonical` folds CLI spellings to one id (`claude-sonnet-5-5`, `anthropic/claude-sonnet-5.5` → `claude-sonnet-5.5`; `openai-codex/gpt-6.1-sol` → `gpt-6.1-sol`; `auto` for a runtime auto-select) and `effort` holds a stripped reasoning-effort suffix (`gemini-3.8-flash-low` → `low`). `provider` is the vendor of the canonical family, never a CLI or gateway name. Rules: `pkg/modelfamily.Resolve`. | | `sandbox_runtime` | Stamped on new bundles from the agent's effective crux sandbox (`podman`, `kubernetes-job`, …) or `none` when agents run in-process. Old bundles fall back to the current config. | | `task.kind` | Stamped from the agent's lane (`docs`, `security-fix`, `ci-repair`, `release`, `dependency-update`, `triage`, `refactor`), else `unknown`. Reviewer records are `pr-review`. | | `task.acmm_level` | `policy.acmm_level` at review time. | diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go index 6f52f14ab5..f3f1fcf001 100644 --- a/src/pkg/evidence/benchmarks/convert.go +++ b/src/pkg/evidence/benchmarks/convert.go @@ -7,6 +7,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/evidence" + "github.com/hivecommons/hive/pkg/modelfamily" ) // Options parameterise one export. @@ -65,12 +66,6 @@ var kindHints = []struct{ hint, kind string }{ {"dep", "dependency-update"}, {"renovate", "dependency-update"}, {"triage", "triage"}, {"refactor", "refactor"}, } -var providerPrefixes = []struct{ prefix, provider string }{ - {"claude", "anthropic"}, {"gpt", "openai"}, {"o1", "openai"}, {"o3", "openai"}, {"o4", "openai"}, - {"gemini", "google"}, {"granite", "ibm"}, {"llama", "meta"}, {"mistral", "mistral"}, - {"qwen", "alibaba"}, {"deepseek", "deepseek"}, {"grok", "xai"}, {"kimi", "moonshot"}, -} - // Convert folds bundles per PR and emits an author record for each PR a hive // agent opened and a reviewer record for each PR the swarm scored. Output is // sorted by id so exports are byte-stable. @@ -133,7 +128,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { name = "agent" } rec.Agent = Agent{Name: name, Role: RoleAuthor, Harness: harnessOf(a.Backend)} - rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)} + rec.Model = modelOf(a.Model) rec.SandboxRuntime = runtimeOf(opts, a, name) rec.Task.Kind = kindOf(a.TaskKind, name) rec.Cost = costOf(opts.CostLookup, authorCostRefs(latest)) @@ -143,7 +138,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec := base rec.ID = recordID(hash, latest.Repo, latest.Number, RoleReviewer) rec.Agent = Agent{Name: reviewSwarmAgent, Role: RoleReviewer, Harness: harnessOf(backend)} - rec.Model = Model{Provider: providerOf(model), Name: model} + rec.Model = modelOf(model) rec.SandboxRuntime = SandboxRuntimeNone rec.Task.Kind = kindPRReview rec.Cost = costOf(opts.CostLookup, prCostRefs(latest.Repo, latest.Number, costKindPR)) @@ -367,17 +362,15 @@ func harnessOf(backend string) string { return harnessOther } -func providerOf(model string) string { - m := strings.ToLower(strings.TrimSpace(model)) - if p, _, ok := strings.Cut(m, "/"); ok && p != "" { - return p - } - for _, pp := range providerPrefixes { - if strings.HasPrefix(m, pp.prefix) { - return pp.provider - } +// modelOf keeps the reported id and adds its canonical reading so one model +// aggregates under one row whichever CLI reported it. +func modelOf(reported string) Model { + r := modelfamily.Resolve(reported, "") + provider := r.Vendor + if provider == modelfamily.Unknown { + provider = providerUnknown } - return providerUnknown + return Model{Provider: provider, Name: strings.TrimSpace(reported), Canonical: r.Canonical, Effort: r.Effort} } // kindOf prefers the task kind stamped on the bundle and otherwise guesses diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go index 6bef3b93b8..b9c91c8e27 100644 --- a/src/pkg/evidence/benchmarks/convert_test.go +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -331,3 +331,20 @@ func TestConvertEdgeCases(t *testing.T) { t.Errorf("qualityOf with nothing to report = %+v, want nil", got) } } + +func TestConvertCanonicalisesReportedModel(t *testing.T) { + bundles := fixtureBundles() + bundles[1].Author.Model = "openai-codex/gpt-5.6-sol:high" + for i := range bundles[1].Verdicts { + bundles[1].Verdicts[i].Model = "claude-sonnet-5-5" + } + recs := Convert(bundles, testOptions()) + a := byRole(recs, "hivecommons/hive", RoleAuthor) + if a == nil || a.Model.Name != "openai-codex/gpt-5.6-sol:high" || a.Model.Canonical != "gpt-5.6-sol" || a.Model.Provider != "openai" || a.Model.Effort != "high" { + t.Errorf("author model not canonicalised: %+v", a) + } + r := byRole(recs, "hivecommons/hive", RoleReviewer) + if r == nil || r.Model.Name != "claude-sonnet-5-5" || r.Model.Canonical != "claude-sonnet-5.5" || r.Model.Provider != "anthropic" || r.Model.Effort != "" { + t.Errorf("reviewer model not canonicalised: %+v", r) + } +} diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go index 875297febd..9e985f1f3a 100644 --- a/src/pkg/evidence/benchmarks/record.go +++ b/src/pkg/evidence/benchmarks/record.go @@ -68,10 +68,16 @@ type Agent struct { Harness string `json:"harness"` } -// Model is the model the agent ran on. +// Model is the model the agent ran on. Name is the id exactly as the +// harness reported it; Canonical folds the spellings different CLIs use for +// one model ("claude-sonnet-5-5", "anthropic/claude-sonnet-5.5" → +// "claude-sonnet-5.5", modelfamily.Resolve) so the leaderboard shows one +// row per model, and Provider is the vendor derived the same way. type Model struct { - Provider string `json:"provider"` - Name string `json:"name"` + Provider string `json:"provider"` + Name string `json:"name"` + Canonical string `json:"canonical,omitempty"` + Effort string `json:"effort,omitempty"` } // Task describes the unit of work. diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json index 87f9edd6bb..a946e89d0d 100644 --- a/src/pkg/evidence/benchmarks/testdata/result.schema.json +++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json @@ -142,12 +142,16 @@ "type": "string", "description": "Exact model id as reported by the harness (evidence author.model)." }, + "canonical": { + "type": "string", + "description": "Canonical model id the reported name folds to (pkg/modelfamily.Resolve): lowercase, CLI/gateway prefix and reasoning-effort suffix stripped, version separators as dots, e.g. claude-sonnet-5.5; 'auto' for a runtime auto-select." + }, "version": { "type": "string" }, "effort": { "type": "string", - "description": "Reasoning effort / thinking budget label if the harness exposes one." + "description": "Reasoning effort / thinking budget label if the harness exposes one (low, medium, high, xhigh, max); split off the model id by the exporter." } } }, diff --git a/src/pkg/modelfamily/modelfamily.go b/src/pkg/modelfamily/modelfamily.go index 7cf5b683a0..4a27eb75f0 100644 --- a/src/pkg/modelfamily/modelfamily.go +++ b/src/pkg/modelfamily/modelfamily.go @@ -43,29 +43,170 @@ func Normalize(value string) string { // ("openai-codex/gpt-6.1-sol" → "gpt-6.1-sol", "anthropic/claude-opus-5" → // "claude-opus-5"), a ":variant" suffix ("gpt-5.6-sol:high" → "gpt-5.6-sol") // and a dash between two digits ("claude-opus-5-5" → "claude-opus-5.5"). -// Unknown stays Unknown. +// Unknown stays Unknown. Reasoning-effort suffixes are kept; Resolve strips +// them. func Canonical(model string) string { m := Normalize(model) if m == Unknown { return Unknown } + m = canonicalID(m) + if m == "" { + return Unknown + } + return m +} + +// Auto is the canonical id and vendor of a runtime's auto-selected model: +// the harness picked, the hive was not told which. +const Auto = "auto" + +// Resolved is the canonical reading of a raw model id. +type Resolved struct { + // Canonical is the id spellings fold to ("claude-sonnet-5.5"); Auto for + // a runtime auto-select; Unknown when nothing was reported. + Canonical string + // Vendor is the model's vendor family ("anthropic", "openai", ...), + // inferred from the model name when the reported prefix is a CLI or + // gateway ("openai-codex/", "aws/"); Auto for Auto, Unknown otherwise. + Vendor string + // Effort is the reasoning-effort label stripped from the id + // ("gemini-3.8-flash-low" → "low", "gpt-5.6-sol:high" → "high"); empty + // when none was reported. + Effort string +} + +// effortLabels are the reasoning-effort suffixes harnesses glue onto ids. +var effortLabels = map[string]bool{"low": true, "medium": true, "high": true, "xhigh": true, "max": true} + +// gateways are prefixes that name the CLI or routing layer, not the vendor, +// so they never become Resolved.Vendor. +var gateways = map[string]bool{ + "openai-codex": true, "github-copilot": true, "copilot": true, "openrouter": true, "aws": true, "bedrock": true, + "vertex": true, "azure": true, "rits": true, "pi-claude-code-provider": true, "kilocode": true, "lab-worker": true, + "stealth": true, +} + +// vendorFamilies maps a canonical id's family (text before the first dash) +// to its vendor. +var vendorFamilies = map[string]string{ + "claude": "anthropic", "gpt": "openai", "o1": "openai", "o3": "openai", "o4": "openai", "codex": "openai", + "gemini": "google", "gemma": "google", "granite": "ibm", "llama": "meta", "mistral": "mistral", "qwen": "alibaba", + "deepseek": "deepseek", "grok": "xai", "kimi": "moonshot", "glm": "z-ai", "minimax": "minimax", "mimo": "xiaomi", + "muse": "meta", +} + +// canonicalOverrides are ids that do not follow the generic rules. +var canonicalOverrides = map[string]string{ + "sonnet": "claude-sonnet", "opus": "claude-opus", "haiku": "claude-haiku", "default": Auto, +} + +// Resolve reads a raw model id into its canonical id, vendor and effort. +// reportedProvider is the vendor the harness claimed, if any; it is used +// only when the model name itself does not identify a vendor and it is not +// a CLI or gateway name. +func Resolve(model, reportedProvider string) Resolved { + raw := strings.TrimSpace(strings.ToLower(model)) + raw = strings.Trim(raw, "`'\"“”‘’") + if raw == Auto || raw == "default" { + return Resolved{Canonical: Auto, Vendor: Auto} + } + m := Normalize(raw) + if m == Unknown { + return Resolved{Canonical: Unknown, Vendor: Unknown} + } + prefix := "" + if i := strings.Index(m, "/"); i >= 0 { + prefix = m[:i] + } + var effort string + if i := strings.Index(m, ":"); i >= 0 { + if effortLabels[m[i+1:]] { + effort = m[i+1:] + } + } + id := canonicalID(m) + if parts := strings.Split(id, "-"); len(parts) > 1 && effortLabels[parts[len(parts)-1]] { + if effort == "" { + effort = parts[len(parts)-1] + } + id = strings.Join(parts[:len(parts)-1], "-") + } + if o, ok := canonicalOverrides[id]; ok { + id = o + } + if id == "" { + return Resolved{Canonical: Unknown, Vendor: Unknown, Effort: effort} + } + if id == Auto { + return Resolved{Canonical: Auto, Vendor: Auto, Effort: effort} + } + vendor := Unknown + family := id + if i := strings.Index(id, "-"); i >= 0 { + family = id[:i] + } + if v, ok := vendorFamilies[family]; ok { + vendor = v + } else if prefix != "" && !gateways[prefix] { + vendor = prefix + } else if p := strings.TrimSpace(strings.ToLower(reportedProvider)); p != "" && p != Unknown && !gateways[p] { + vendor = p + } + return Resolved{Canonical: id, Vendor: vendor, Effort: effort} +} + +// canonicalID applies the spelling folds to an already-normalized id: drop +// path prefixes and ":variant", spaces and underscores to dashes, join +// consecutive all-digit segments with dots ("sonnet-5-5" → "sonnet-5.5" but +// "ornith-1.5-35b" stays), drop an 8-digit date stamp ("-20250929"). +func canonicalID(m string) string { if i := strings.LastIndex(m, "/"); i >= 0 { m = m[i+1:] } if i := strings.Index(m, ":"); i >= 0 { m = m[:i] } - b := []byte(m) - for i := 1; i+1 < len(b); i++ { - if b[i] == '-' && isDigit(b[i-1]) && isDigit(b[i+1]) { - b[i] = '.' + m = strings.NewReplacer(" ", "-", "_", "-").Replace(m) + for strings.Contains(m, "--") { + m = strings.ReplaceAll(m, "--", "-") + } + parts := strings.Split(strings.Trim(m, "-"), "-") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if p == "" { + continue + } + if len(p) == 8 && allDigits(p) && len(out) > 0 { + continue } + if n := len(out); n > 0 && allDigits(p) && allDigits(lastSegment(out[n-1])) { + out[n-1] += "." + p + continue + } + out = append(out, p) } - m = strings.TrimSpace(string(b)) - if m == "" { - return Unknown + return strings.Join(out, "-") +} + +// lastSegment is the text after the final dot, so "5.5" reads as digits. +func lastSegment(s string) string { + if i := strings.LastIndex(s, "."); i >= 0 { + return s[i+1:] } - return m + return s +} + +func allDigits(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if !isDigit(s[i]) { + return false + } + } + return true } func isDigit(c byte) bool { return c >= '0' && c <= '9' } diff --git a/src/pkg/modelfamily/modelfamily_test.go b/src/pkg/modelfamily/modelfamily_test.go index 39061e481c..a052b41fa4 100644 --- a/src/pkg/modelfamily/modelfamily_test.go +++ b/src/pkg/modelfamily/modelfamily_test.go @@ -38,6 +38,8 @@ func TestCanonicalFoldsSpellings(t *testing.T) { "lemonade/qwen-3": "qwen-3", "": Unknown, "auto": Unknown, + "-": Unknown, + "gemini-3.8-flash-low": "gemini-3.8-flash-low", } for in, want := range cases { if got := Canonical(in); got != want { @@ -57,3 +59,50 @@ func TestSameFamily(t *testing.T) { t.Fatalf("unknown author: got same=%v known=%v", same, known) } } + +func TestResolveFoldsRuntimeSpellings(t *testing.T) { + cases := []struct { + in, provider string + want Resolved + }{ + {"claude-sonnet-5-5", "anthropic", Resolved{Canonical: "claude-sonnet-5.5", Vendor: "anthropic"}}, + {"claude-sonnet-5.5", "anthropic", Resolved{Canonical: "claude-sonnet-5.5", Vendor: "anthropic"}}, + {"openai-codex/gpt-6.1-sol", "openai-codex", Resolved{Canonical: "gpt-6.1-sol", Vendor: "openai"}}, + {"gpt-6.1-sol", "openai", Resolved{Canonical: "gpt-6.1-sol", Vendor: "openai"}}, + {"auto", "unknown", Resolved{Canonical: Auto, Vendor: Auto}}, + {"default", "", Resolved{Canonical: Auto, Vendor: Auto}}, + {"claude-opus-4-6", "", Resolved{Canonical: "claude-opus-4.6", Vendor: "anthropic"}}, + {"claude-opus-4.6", "", Resolved{Canonical: "claude-opus-4.6", Vendor: "anthropic"}}, + {"gemini-3.8-flash-low", "google", Resolved{Canonical: "gemini-3.8-flash", Vendor: "google", Effort: "low"}}, + {"anthropic/claude-sonnet-5:max", "anthropic", Resolved{Canonical: "claude-sonnet-5", Vendor: "anthropic", Effort: "max"}}, + {"openai-codex/gpt-5.6-sol:high", "openai-codex", Resolved{Canonical: "gpt-5.6-sol", Vendor: "openai", Effort: "high"}}, + {"aws/claude-haiku-4-5", "aws", Resolved{Canonical: "claude-haiku-4.5", Vendor: "anthropic"}}, + {"github-copilot/gemini-3.8-flash", "github-copilot", Resolved{Canonical: "gemini-3.8-flash", Vendor: "google"}}, + {"rits/zai-org/glm-5-3", "rits", Resolved{Canonical: "glm-5.3", Vendor: "z-ai"}}, + {"lemonade/ornith-1.5-35b-a3b-gguf-q6_k", "lemonade", Resolved{Canonical: "ornith-1.5-35b-a3b-gguf-q6-k", Vendor: "lemonade"}}, + {"pi-claude-code-provider/sonnet", "pi-claude-code-provider", Resolved{Canonical: "claude-sonnet", Vendor: "anthropic"}}, + {"GPT-5.6 Terra", "openai", Resolved{Canonical: "gpt-5.6-terra", Vendor: "openai"}}, + {"claude-sonnet-4-5-20250929", "", Resolved{Canonical: "claude-sonnet-4.5", Vendor: "anthropic"}}, + {"kimi-k3", "moonshot", Resolved{Canonical: "kimi-k3", Vendor: "moonshot"}}, + {"unknown", "unknown", Resolved{Canonical: Unknown, Vendor: Unknown}}, + {"", "", Resolved{Canonical: Unknown, Vendor: Unknown}}, + {"ornith-1.5", " Lemonade ", Resolved{Canonical: "ornith-1.5", Vendor: "lemonade"}}, + {"ornith-1.5", "aws", Resolved{Canonical: "ornith-1.5", Vendor: Unknown}}, + {"ornith-1.5", "unknown", Resolved{Canonical: "ornith-1.5", Vendor: Unknown}}, + {"ornith-1.5", "", Resolved{Canonical: "ornith-1.5", Vendor: Unknown}}, + {"-", "", Resolved{Canonical: Unknown, Vendor: Unknown}}, + {"-:high", "", Resolved{Canonical: Unknown, Vendor: Unknown, Effort: "high"}}, + {"default:high", "", Resolved{Canonical: Auto, Vendor: Auto, Effort: "high"}}, + {"claude--opus-high", "", Resolved{Canonical: "claude-opus", Vendor: "anthropic", Effort: "high"}}, + {"opus", "", Resolved{Canonical: "claude-opus", Vendor: "anthropic"}}, + {"haiku:low", "", Resolved{Canonical: "claude-haiku", Vendor: "anthropic", Effort: "low"}}, + {"gpt-5.6-sol:fast", "openai", Resolved{Canonical: "gpt-5.6-sol", Vendor: "openai"}}, + {"gemini-3.8-flash-low:high", "google", Resolved{Canonical: "gemini-3.8-flash", Vendor: "google", Effort: "high"}}, + {"`'Claude-Opus-4-6'`", "", Resolved{Canonical: "claude-opus-4.6", Vendor: "anthropic"}}, + } + for _, c := range cases { + if got := Resolve(c.in, c.provider); got != c.want { + t.Errorf("Resolve(%q,%q)=%+v want %+v", c.in, c.provider, got, c.want) + } + } +} From 829b93e12e56e800910a3d06badcb5d275d75696 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:13:52 -0500 Subject: [PATCH 137/223] =?UTF-8?q?=E2=9C=A8=20activity=20monitor:=20stage?= =?UTF-8?q?-meaning=20icons=20in=20inactive=20workflow=20chevrons,=20optio?= =?UTF-8?q?nal=20column=20icon=20override=20(#18038)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ activity monitor: stage-meaning icons in inactive workflow chevrons, optional column icon override Every column across hive-default, the curated marketplace bundles, the ingest skeleton and the editor palette resolves to a monochrome inline-SVG glyph by its normalised name (unknown → a neutral dot). Done / todo model chevrons draw the glyph alone at 60%; the active one leads its word with it. spec.columns[].icon overrides the glyph with an allowlisted id or a single emoji, validated in workflowmodel.Validate and in the editor, so no markup reaches a chevron. The editor's column inspector previews the icon and offers a pick-list plus an emoji box. Fixes #18033 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 activity monitor: fix CI for chevron stage icons - lift stageTrackerWorkflowColumnIconHtml into the workflow-columns node fixture so stageTrackerWorkflowStripItems resolves it - ValidateColumnIcon trims only blanks, so a trailing newline is rejected - TestValidateRejectsBadColumnIcon uses the techdocs fixture (hive-default has no columns) - chevron / editor emoji glyphs size with --fs tokens instead of raw px so the style ratchet stays at its baseline - drop the '#' from the tracker.css comment that sits inside the repo-chip tokens-only guard window Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-chevron-stage-icons.md | 1 + src/pkg/dashboard/api_workflows_test.go | 24 ++ .../stage_tracker_stage_icons_ui_test.go | 123 ++++++++++ src/pkg/dashboard/stage_tracker_workflows.go | 4 +- .../stage_tracker_workflows_ui_test.go | 2 +- src/pkg/dashboard/static-src/css/tracker.css | 8 + .../static-src/css/workflows-editor.css | 6 + .../static-src/js/config-workflows-editor.js | 32 +++ .../static-src/js/tracker-stage-icons.js | 160 +++++++++++++ .../static-src/js/tracker-workflows.js | 16 +- src/pkg/dashboard/static-src/manifest.txt | 1 + src/pkg/dashboard/static/index.html | 222 +++++++++++++++++- src/pkg/workflowmodel/icons.go | 97 ++++++++ src/pkg/workflowmodel/icons_test.go | 60 +++++ src/pkg/workflowmodel/lifecycle.go | 10 +- src/pkg/workflowmodel/model.go | 5 + 16 files changed, 762 insertions(+), 9 deletions(-) create mode 100644 changelog.d/added-chevron-stage-icons.md create mode 100644 src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go create mode 100644 src/pkg/dashboard/static-src/js/tracker-stage-icons.js create mode 100644 src/pkg/workflowmodel/icons.go create mode 100644 src/pkg/workflowmodel/icons_test.go diff --git a/changelog.d/added-chevron-stage-icons.md b/changelog.d/added-chevron-stage-icons.md new file mode 100644 index 0000000000..341298d6e9 --- /dev/null +++ b/changelog.d/added-chevron-stage-icons.md @@ -0,0 +1 @@ +- Activity monitor: a workflow-model chevron that is not the row's active stage now wears a glyph for what the stage means (inbox, bolt, hand, pencil, pull-request, flask, eye, people, thumbs-up, merge, check, …) picked by the column's name, dimmed to 60%, instead of its clipped label; the active chevron leads its word with the same glyph. Workflow columns take an optional `icon` (an allowlisted icon id or a single emoji, validated server- and client-side; never markup), and the workflow editor's column inspector previews the default and lets the operator replace it (#18033). diff --git a/src/pkg/dashboard/api_workflows_test.go b/src/pkg/dashboard/api_workflows_test.go index 7a7a7d9c59..1b106149eb 100644 --- a/src/pkg/dashboard/api_workflows_test.go +++ b/src/pkg/dashboard/api_workflows_test.go @@ -378,3 +378,27 @@ func TestWorkflowEnabledPut(t *testing.T) { t.Fatalf("no config: %d", rec.Code) } } + +// A column's `icon` rides the tracker block as the SPA reads it (trimmed), +// and a model whose column names a non-allowlisted icon is refused at the +// API, so a chevron can only ever draw an allowlisted glyph +// (hivecommons/hive#18033). +func TestStageTrackerWorkflowsColumnIcon(t *testing.T) { + m := techdocsModel(t) + m.Spec.Columns[0].Icon = " bolt " + m.Spec.Columns[1].Icon = "📝" + p, _, err := workflowmodel.Compile(m) + if err != nil { + t.Fatal(err) + } + blk := stageTrackerWorkflowBlock(p) + if blk.Columns[0].Icon != "bolt" || blk.Columns[1].Icon != "📝" || blk.Columns[2].Icon != "" { + t.Fatalf("icons not carried: %+v", blk.Columns) + } + s := workflowTestServer(t, true) + bad := techdocsModel(t) + bad.Spec.Columns[0].Icon = "<svg onload=alert(1)>" + if rec := wfReq(s, http.MethodPost, "/api/workflows", bad, true); rec.Code == http.StatusCreated { + t.Fatalf("model with markup icon accepted: %s", rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go b/src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go new file mode 100644 index 0000000000..be262eb2a6 --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go @@ -0,0 +1,123 @@ +package dashboard + +import ( + "os" + "os/exec" + "regexp" + "sort" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// The SPA's icon set and the server's allowlist are one list +// (hivecommons/hive#18033): an id the server accepts must draw, and the +// SPA must not draw an id the server would refuse. +func TestStageIconSetMatchesAllowlist(t *testing.T) { + html := indexHTML(t) + block := jsConstBlock(t, html, "STAGE_ICON_SET") + keyRe := regexp.MustCompile(`(?m)^\s*'?([a-z][a-z-]*)'?: '`) + var js []string + for _, m := range keyRe.FindAllStringSubmatch(block, -1) { + js = append(js, m[1]) + } + want := append([]string{}, workflowmodel.ColumnIconIDs...) + sort.Strings(js) + sort.Strings(want) + if strings.Join(js, ",") != strings.Join(want, ",") { + t.Errorf("STAGE_ICON_SET ids %v != workflowmodel.ColumnIconIDs %v", js, want) + } + if !strings.Contains(html, `const STAGE_ICON_DEFAULT = '`+workflowmodel.ColumnIconDefault+`';`) { + t.Errorf("STAGE_ICON_DEFAULT is not %q", workflowmodel.ColumnIconDefault) + } + for _, want := range []string{ + `js/tracker-stage-icons.js`, + `function stageIconColumnHtml(col, cls, label)`, + `function wfEdColumnIconField(c, base, dis)`, + `function wfEdSetColumnIcon(path, value)`, + `.ham-seg.ham-wf-seg.done .ham-stage-ico { opacity: 0.6; }`, + `.ham-emoji-ico {`, + } { + if !strings.Contains(html, want) && !strings.Contains(manifestText(t), want) { + t.Errorf("index.html missing stage-icon wiring %q", want) + } + } +} + +func manifestText(t *testing.T) string { + t.Helper() + b, err := os.ReadFile("static-src/manifest.txt") + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// Under node: names normalise and resolve (unknown → the neutral dot), an +// override must be an allowlisted id or one emoji (markup and words are +// refused and never rendered), and the model strip draws the icon alone +// in done / todo chevrons and ahead of the word in the active one. +func TestStageIconsNode(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not installed") + } + html := indexHTML(t) + script := `const assert = require('assert'); +const STAGE_TRACKER_TERMINAL = 110; +const STAGE_TRACKER_WORKFLOW_UNCOVERED_WORD = 'Not covered'; +function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); } +` + jsConstBlock(t, html, "STAGE_ICON_SET") + "\n" + jsConstBlock(t, html, "STAGE_ICON_BY_NAME") + "\nconst STAGE_ICON_DEFAULT = 'dot';\n" + + jsFunc(t, html, "stageIconNormalizeName") + jsFunc(t, html, "stageIconIsId") + jsFunc(t, html, "stageIconIsEmoji") + + jsFunc(t, html, "stageIconIsValid") + jsFunc(t, html, "stageIconIdForName") + jsFunc(t, html, "stageIconFor") + + jsFunc(t, html, "stageIconHtml") + jsFunc(t, html, "stageIconColumnHtml") + + jsFunc(t, html, "stageTrackerWorkflowColumnIconHtml") + jsFunc(t, html, "stageTrackerWorkflowColumnIndex") + + jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowHeaderHtml") + ` +// normalisation and the canonical map +assert.strictEqual(stageIconNormalizeName(' Reviewer Check '), 'reviewer check'); +assert.strictEqual(stageIconNormalizeName('CI / Review'), 'ci review'); +assert.strictEqual(stageIconNormalizeName('Deployed · staging'), 'deployed staging'); +const expect = { 'Open': 'inbox', 'Actionable': 'bolt', 'Claimed': 'hand', 'PR in progress': 'pull-request', 'CI / Review': 'flask', 'Review': 'eye', 'Merge queue': 'merge', 'Merged': 'check', + 'Requested': 'inbox', 'Accepted': 'checkbox', 'Drafting': 'pencil', 'Reviewer check': 'eye', 'Stakeholder review': 'people', 'Approval': 'thumbs-up', 'Done': 'check', + 'Intake': 'inbox', 'Triage': 'tag', 'Analysis': 'search', 'Writing': 'pencil', 'Publish': 'rocket', 'No such stage': 'dot' }; +for (const k of Object.keys(expect)) assert.strictEqual(stageIconIdForName(k), expect[k], k); +for (const id of Object.values(STAGE_ICON_BY_NAME)) assert.ok(stageIconIsId(id), 'map names unknown id ' + id); +// the allowlist +for (const ok of ['', 'bolt', ' merge ', '✅', '🚀', '👍🏽', '👩‍💻', '🇺🇸']) assert.ok(stageIconIsValid(ok), 'valid ' + ok); +for (const bad of ['Bolt', 'sparkle', '<svg>', '<img src=x onerror=alert(1)>', 'a', '✅✅✅', 'ok ✅', 'javascript:']) assert.ok(!stageIconIsValid(bad), 'invalid ' + bad); +assert.strictEqual(stageIconFor({ label: 'Drafting', icon: '<b>x</b>' }), 'pencil', 'invalid override falls back to the label'); +assert.strictEqual(stageIconFor({ label: 'Drafting', icon: 'rocket' }), 'rocket'); +assert.strictEqual(stageIconFor({ label: 'Drafting', icon: '🚀' }), '🚀'); +assert.strictEqual(stageIconHtml('<svg onload=alert(1)>', 'c'), '', 'markup never renders'); +assert.strictEqual(stageIconHtml('sparkle', 'c'), ''); +const svg = stageIconHtml('bolt', 'ham-ico ham-stage-ico'); +assert.ok(svg.startsWith('<svg class="ham-ico ham-stage-ico"') && svg.includes('stroke="currentColor"') && svg.includes('aria-hidden="true"')); +assert.ok(stageIconHtml('bolt', 'c', 'Actionable').includes('role="img" aria-label="Actionable"')); +const emo = stageIconHtml('🚀', 'ham-ico'); +assert.ok(emo.includes('ham-emoji-ico') && emo.includes('🚀') && !emo.includes('<svg')); +// the strip: icon alone when inactive, icon + word when active +const wf = { name: 't', version: '1', columns: [ + { id: 'a', label: 'Requested', states: [], numbers: [10] }, + { id: 'b', label: 'Drafting', states: [], numbers: [20], icon: '🚀' }, + { id: 'c', label: 'Review', states: [], numbers: [30], icon: '<script>' }, + { id: 'd', label: 'Done', terminal: true, states: [], numbers: [] }, +]}; +const lis = stageTrackerWorkflowStripItems({ stage: 20 }, wf, []).split('</li>').filter(Boolean); +assert.strictEqual(lis.length, 4); +assert.ok(lis[0].includes(' done') && lis[0].includes('<svg') && lis[0].includes('ham-stage-ico') && !lis[0].includes('ham-seg-t'), 'done chevron = icon alone: ' + lis[0]); +assert.ok(lis[1].includes(' active') && lis[1].includes('ham-emoji-ico') && lis[1].includes('<span class="ham-seg-t">Drafting</span>'), 'active = icon + word: ' + lis[1]); +assert.ok(lis[2].includes(' todo') && lis[2].includes(STAGE_ICON_SET.eye) && !lis[2].includes('<script>'), 'invalid override falls back and never renders: ' + lis[2]); +assert.ok(lis[3].includes(' todo') && lis[3].includes('cap') && lis[3].includes(STAGE_ICON_SET.check)); +assert.ok(lis[0].includes('title="Requested"'), 'label stays the hover'); +// header keeps the label and gains the icon +const head = stageTrackerWorkflowHeaderHtml(wf); +assert.ok(head.includes('ham-col-ico') && head.includes('Requested') && head.includes('Done')); +console.log('ok'); +` + cmd := exec.Command(node, "-e", script) + out, err := cmd.CombinedOutput() + if err != nil || !strings.Contains(string(out), "ok") { + t.Fatalf("node: %v\n%s", err, out) + } +} diff --git a/src/pkg/dashboard/stage_tracker_workflows.go b/src/pkg/dashboard/stage_tracker_workflows.go index 639739a670..9b3b3fe83f 100644 --- a/src/pkg/dashboard/stage_tracker_workflows.go +++ b/src/pkg/dashboard/stage_tracker_workflows.go @@ -3,6 +3,7 @@ package dashboard import ( "sort" "strconv" + "strings" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/workflowmodel" @@ -45,6 +46,7 @@ type StageTrackerWorkflowColumn struct { ID string `json:"id"` Label string `json:"label"` Hint string `json:"hint,omitempty"` + Icon string `json:"icon,omitempty"` Terminal bool `json:"terminal,omitempty"` States []string `json:"states"` Numbers []int `json:"numbers"` @@ -57,7 +59,7 @@ type workflowProfileLookup func(name, version string) (*workflowmodel.Profile, e func stageTrackerWorkflowBlock(p *workflowmodel.Profile) StageTrackerWorkflow { out := StageTrackerWorkflow{Name: p.Name, Version: p.Version, Hash: p.Hash} for _, c := range p.Columns { - col := StageTrackerWorkflowColumn{ID: c.ID, Label: c.Label, Hint: c.Hint, Terminal: c.Terminal, States: append([]string{}, c.States...), Numbers: []int{}} + col := StageTrackerWorkflowColumn{ID: c.ID, Label: c.Label, Hint: c.Hint, Icon: strings.TrimSpace(c.Icon), Terminal: c.Terminal, States: append([]string{}, c.States...), Numbers: []int{}} for _, id := range c.States { if st := p.States[id]; st != nil && st.Lifecycle.Number > 0 { col.Numbers = append(col.Numbers, st.Lifecycle.Number) diff --git a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go index 3b9437a5f1..401ec67714 100644 --- a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go +++ b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go @@ -59,7 +59,7 @@ const STAGE_TRACKER_TERMINAL = 110; function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); } ` + jsFunc(t, html, "stageTrackerWorkflowOf") + jsFunc(t, html, "stageTrackerWorkflowColumnIndex") + jsFunc(t, html, "stageTrackerWorkflowBadgeHtml") + jsFunc(t, html, "stageTrackerWorkflowHeaderHtml") + - jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowKey") + + jsFunc(t, html, "stageTrackerWorkflowColumnIconHtml") + jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowKey") + jsFunc(t, html, "stageTrackerColumnsHeaderHtml") + jsFunc(t, html, "stageTrackerWorkflowRuns") + jsFunc(t, html, "stageTrackerWorkflowRunHeaderRows") + jsFunc(t, html, "stageTrackerGroupColumnsHtml") + ` const STAGE_TRACKER_WORKFLOW_UNCOVERED_WORD = 'Not covered'; diff --git a/src/pkg/dashboard/static-src/css/tracker.css b/src/pkg/dashboard/static-src/css/tracker.css index 53a8eadfc8..e9411e4da6 100644 --- a/src/pkg/dashboard/static-src/css/tracker.css +++ b/src/pkg/dashboard/static-src/css/tracker.css @@ -272,6 +272,14 @@ .ham-wf-badge { flex: 0 0 auto; font-family: var(--font-mono); font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; color: var(--indigo); background: color-mix(in srgb, var(--indigo) 12%, transparent); border: 1px solid color-mix(in srgb, var(--indigo) 40%, transparent); } .ham-wf-cols { position: static; } .ham-wf-cols .ham-wf-col { color: var(--indigo); } + /* Model chevrons wear the column's stage icon (hivecommons/hive issue 18033): + alone and dimmed in a done / todo step, leading the word in the lit + one. An emoji override is sized to the SVG glyph so the row keeps + its height. */ + .ham-seg.ham-wf-seg .ham-stage-ico { flex: 0 0 auto; } + .ham-seg.ham-wf-seg.active .ham-stage-ico { margin-right: var(--sp-1); } + .ham-seg.ham-wf-seg.done .ham-stage-ico { opacity: 0.6; } + .ham-emoji-ico { display: inline-block; width: 12px; height: 12px; font-size: var(--fs-xs); line-height: 12px; text-align: center; font-style: normal; } .ham-seg.ham-wf-uncovered { background: color-mix(in srgb, var(--amber) 18%, transparent); color: var(--amber); outline: 1px dashed var(--amber); } .ham-group-dist { flex: 0 0 auto; margin-left: auto; margin-right: calc(4em + var(--sp-3)); display: inline-flex; align-items: flex-end; gap: var(--sp-1); height: 0.9em; } .ham-gd { display: inline-block; width: 0.45em; height: 15%; border-radius: var(--sp-1); background: color-mix(in srgb, var(--text-faint) 35%, transparent); } diff --git a/src/pkg/dashboard/static-src/css/workflows-editor.css b/src/pkg/dashboard/static-src/css/workflows-editor.css index 7d8276cd3d..3189f213fc 100644 --- a/src/pkg/dashboard/static-src/css/workflows-editor.css +++ b/src/pkg/dashboard/static-src/css/workflows-editor.css @@ -108,6 +108,12 @@ .wf-ed-field input[readonly] { color: var(--muted); } .wf-ed-inline { display: flex; gap: var(--sp-2); align-items: center; } .wf-ed-inline input { flex: 1 1 auto; min-width: 0; } + /* Column icon row (hivecommons/hive#18033): preview glyph, id pick-list, emoji box. */ + .wf-ed-icon-row .wf-ed-icon-preview { display: inline-flex; align-items: center; justify-content: center; width: 24px; height: 24px; flex: 0 0 auto; border-radius: var(--r-sm, 4px); background: var(--panel-strong); color: var(--text); } + .wf-ed-icon-row .wf-ed-icon-preview .ham-ico { width: 14px; height: 14px; } + .wf-ed-icon-row .wf-ed-icon-preview .ham-emoji-ico { width: 14px; height: 14px; font-size: var(--fs-sm); line-height: 14px; } + .wf-ed-icon-row select { flex: 1 1 auto; min-width: 0; } + .wf-ed-icon-row .wf-ed-icon-emoji { flex: 0 0 7em; } .wf-ed-checks { display: flex; gap: var(--sp-4); align-items: center; flex-wrap: wrap; } .wf-ed-warn { color: var(--status-warn); } .wf-ed-translist { margin: var(--sp-0); padding-left: var(--sp-5); font-size: var(--fs-xs); } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-editor.js b/src/pkg/dashboard/static-src/js/config-workflows-editor.js index 6649151761..4dfd898aab 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-editor.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-editor.js @@ -877,6 +877,24 @@ options.map(o => { const v = Array.isArray(o) ? o[0] : o, l = Array.isArray(o) ? o[1] : (o || '\u2014'); return `<option value="${esc(v)}" ${v === value ? 'selected' : ''}>${esc(l)}</option>`; }).join('') + '</select>'; } + // The column's chevron icon (hivecommons/hive#18033): a preview of what + // the Activity Monitor draws, a pick-list of the allowlisted ids with + // the label's default first, and a box for a single emoji. Both write + // spec.columns[id].icon through wfEdSetColumnIcon, which refuses + // anything outside the allowlist. + function wfEdColumnIconField(c, base, dis) { + if (typeof stageIconFor !== 'function') return ''; + const cur = String(c.icon == null ? '' : c.icon).trim(); + const def = stageIconIdForName(c.label || c.id); + const isEmoji = !!cur && !stageIconIsId(cur); + const ids = Object.keys(STAGE_ICON_SET); + const options = [['', 'default \u00b7 ' + def]].concat(ids.map(id => [id, id])); + const preview = `<span class="wf-ed-icon-preview" title="${esc('as drawn in the Activity Monitor')}">${stageIconHtml(stageIconFor(c), 'ham-ico ham-stage-ico')}</span>`; + const select = `<select data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>${options.map(o => `<option value="${esc(o[0])}" ${!isEmoji && o[0] === cur ? 'selected' : ''}>${esc(o[1])}</option>`).join('')}${isEmoji ? `<option value="${esc(cur)}" selected>${esc(cur)} (emoji)</option>` : ''}</select>`; + const emoji = `<input type="text" class="wf-ed-icon-emoji" maxlength="16" placeholder="or an emoji" value="${esc(isEmoji ? cur : '')}" data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>`; + return wfEdField('Icon', `<div class="wf-ed-inline wf-ed-icon-row">${preview}${select}${emoji}</div>`, 'drawn in this column\u2019s chevrons; blank = by label'); + } + function renderWfEdInspector(st) { const m = st.model, spec = m.spec, sel = st.selection; const can = wfEdCanEdit(); @@ -927,6 +945,7 @@ wfEdField('Id', `<div class="wf-ed-inline"><input type="text" value="${esc(c.id)}" readonly><button type="button" class="hv-btn btn-secondary btn-sm" data-action="wfEdRenameColumnAction" data-arg0="${esc(c.id)}" ${dis}>Rename\u2026</button></div>`) + wfEdField('Label', wfEdText(base + '.label', c.label, dis)) + wfEdField('Hint', wfEdText(base + '.hint', c.hint, dis), 'shown as the lane\u2019s tooltip in the Activity Monitor') + + wfEdColumnIconField(c, base, dis) + `<div class="config-field wf-ed-field"><label><input type="checkbox" ${c.terminal ? 'checked' : ''} data-change-action="wfEdFieldSetBool" data-arg0="${esc(base + '.terminal')}" data-arg-types="s,c" ${dis}> Terminal column</label><span class="text-muted fs-xs">states here need no exit</span></div>` + wfEdField('Width', `<input type="range" min="${WF_ED.minColW}" max="${WF_ED.maxColW}" step="${WF_ED.grid}" value="${st.view.laneWidths[c.id] || WF_ED.colW}" data-input-action="wfEdLaneWidth" data-arg0="${esc(c.id)}" data-arg-types="s,v">`) + `<p class="text-muted fs-sm">${(c.states || []).length} state${(c.states || []).length === 1 ? '' : 's'}</p>` + @@ -1498,6 +1517,19 @@ wfEdCommit(wfEdUpdate(_wfEd.model, path, isNaN(n) || n <= 0 ? '' : n)); } function wfEdFieldSetBool(path, checked) { wfEdCommit(wfEdUpdate(_wfEd.model, path, checked ? true : '')); } + // An icon is '' (by label), an allowlisted id or one emoji; anything + // else is refused with a toast and the inspector redraws the last good + // value, so no markup ever reaches a chevron (hivecommons/hive#18033). + function wfEdSetColumnIcon(path, value) { + if (!_wfEd.model) return; + const v = String(value == null ? '' : value).trim(); + if (typeof stageIconIsValid === 'function' && !stageIconIsValid(v)) { + if (typeof showToast === 'function') showToast('Icon must be one of the built-in icon ids or a single emoji.', 'error'); + wfEdRefresh(['inspector']); + return; + } + wfEdCommit(wfEdUpdate(_wfEd.model, path, v), { parts: ['canvas', 'inspector', 'drawer'] }); + } function wfEdSetList(path, value) { wfEdCommit(wfEdUpdate(_wfEd.model, path, String(value || '').split(/[,\s]+/).map(s => s.trim()).filter(Boolean))); } function wfEdSetStateColumn(id, columnId) { wfEdCommit(wfEdMoveState(_wfEd.model, id, columnId === WF_ED_UNASSIGNED ? '' : columnId)); } function wfEdSetInitial(id) { wfEdCommit(wfEdUpdate(_wfEd.model, 'spec.initial', id)); } diff --git a/src/pkg/dashboard/static-src/js/tracker-stage-icons.js b/src/pkg/dashboard/static-src/js/tracker-stage-icons.js new file mode 100644 index 0000000000..df7464dfbb --- /dev/null +++ b/src/pkg/dashboard/static-src/js/tracker-stage-icons.js @@ -0,0 +1,160 @@ +/* ---- Stage icons for workflow-model chevrons (hivecommons/hive#18033) --- + A chevron that is not the row's active stage used to carry only its + clipped column label. Now every column has a glyph that says what the + stage means: picked by the column's name (STAGE_ICON_BY_NAME, keyed + by the normalised label), or by the model's own `icon` on the column + — an allowlisted id below or a single emoji, never markup. The set is + 16-unit monochrome strokes in currentColor, so it follows the theme + like STAGE_TRACKER_ICONS does. Pure helpers: the node harness + (stage_tracker_stage_icons_ui_test.go) lifts them by name, and a Go + test keeps the id list equal to workflowmodel.ColumnIconIDs. */ + const STAGE_ICON_SET = { + inbox: '<path d="M2 9.5h3.5l1 2h3l1-2H14"/><path d="M2 9.5 4 3.5h8l2 6v4H2z"/>', + circle: '<circle cx="8" cy="8" r="5"/>', + bolt: '<path d="M9 1 3 9h4l-1 6 6-8H8z"/>', + hand: '<path d="M4.5 8.5V4a1 1 0 0 1 2 0v3.5M6.5 7V3a1 1 0 0 1 2 0v4M8.5 7V3.5a1 1 0 0 1 2 0V8M10.5 8V5a1 1 0 0 1 2 0v5.5c0 2.5-1.8 4-4.3 4-2 0-3-.9-3.9-2.3L2.6 9.9a1 1 0 0 1 1.6-1.2L4.5 9"/>', + person: '<circle cx="8" cy="5" r="3"/><path d="M2.5 14.5a5.5 5.5 0 0 1 11 0"/>', + pencil: '<path d="M11.5 2.5 13.5 4.5 5.5 12.5 2.5 13.5 3.5 10.5z"/><path d="M9.5 4.5l2 2"/>', + 'pull-request': '<circle cx="4" cy="3" r="1.5"/><circle cx="4" cy="13" r="1.5"/><circle cx="12" cy="13" r="1.5"/><path d="M4 4.5v7M12 11.5V7a2 2 0 0 0-2-2H8"/>', + flask: '<path d="M5.5 1.5h5M6.5 1.5v4.6L2.6 12.9a1 1 0 0 0 .9 1.6h9a1 1 0 0 0 .9-1.6L9.5 6.1V1.5"/><path d="M4.4 10.5h7.2"/>', + gear: '<circle cx="8" cy="8" r="2.2"/><path d="M8 1.5v2M8 12.5v2M1.5 8h2M12.5 8h2M3.4 3.4l1.4 1.4M11.2 11.2l1.4 1.4M3.4 12.6l1.4-1.4M11.2 4.8l1.4-1.4"/>', + eye: '<path d="M1 8s2.5-5 7-5 7 5 7 5-2.5 5-7 5-7-5-7-5z"/><circle cx="8" cy="8" r="2"/>', + people: '<circle cx="5.5" cy="5" r="2.5"/><path d="M1 14a4.5 4.5 0 0 1 9 0"/><path d="M10.5 3a2.5 2.5 0 0 1 0 4.5M12 10a4 4 0 0 1 3 4"/>', + 'thumbs-up': '<path d="M5 7.5 8 1.5c1.5 0 2 1 2 2L9.5 6H13a1.5 1.5 0 0 1 1.5 1.7l-1 5A1.5 1.5 0 0 1 12 14H5z"/><path d="M5 7.5H2v6.5h3"/>', + 'shield-check': '<path d="M8 1.5 13.5 3.5v4c0 3.5-2.5 5.9-5.5 7-3-1.1-5.5-3.5-5.5-7v-4z"/><path d="M5.5 8l1.8 1.8L10.8 6"/>', + merge: '<path stroke-width="2" d="M6 14.5V3M3 6l3-3 3 3M13 14.5c0-3.8-3.4-4.6-7-6.5"/>', + check: '<path d="M3 8l3 3 7-7"/>', + checkbox: '<rect x="2.5" y="2.5" width="11" height="11" rx="1.5"/><path d="M5 8l2 2 4-4"/>', + list: '<path d="M3 5h10M3 8h10M3 11h10"/>', + tag: '<path d="M2.5 2.5h5.5l5.5 5.5-5.5 5.5-5.5-5.5z"/><circle cx="5.5" cy="5.5" r="1"/>', + flag: '<path d="M3.5 14.5V2.5h8.5l-2 3 2 3H3.5"/>', + rocket: '<path d="M8 1.5c2.5 1.5 3.5 4.5 3 8l-3 3-3-3c-.5-3.5.5-6.5 3-8z"/><circle cx="8" cy="6.5" r="1.2"/><path d="M5 9.5 3 11.5l.5 2 2-1M11 9.5l2 2-.5 2-2-1"/>', + book: '<path d="M2.5 2.5h4a1.5 1.5 0 0 1 1.5 1.5v9.5a1 1 0 0 0-1-1h-4.5zM13.5 2.5h-4A1.5 1.5 0 0 0 8 4v9.5a1 1 0 0 1 1-1h4.5z"/>', + search: '<circle cx="7" cy="7" r="4.5"/><path d="M10.5 10.5 14 14"/>', + clock: '<circle cx="8" cy="8" r="6"/><path d="M8 4.5V8l2.5 1.5"/>', + pause: '<path d="M5.5 3v10M10.5 3v10"/>', + archive: '<rect x="2" y="2.5" width="12" height="3"/><path d="M3 5.5v8h10v-8M6.5 8.5h3"/>', + dot: '<circle cx="8" cy="8" r="1.75" fill="currentColor"/>', + }; + const STAGE_ICON_DEFAULT = 'dot'; + + // Pure: the key a column name is matched by — lower-case, trimmed, + // inner whitespace and the " / " and " · " separators collapsed to one + // space, so "Reviewer check", "reviewer check" and "CI / Review" each + // land on one entry. + function stageIconNormalizeName(name) { + return String(name == null ? '' : name).toLowerCase().replace(/[\s·/]+/g, ' ').trim(); + } + + // Canonical stage name → icon id. Every column label across hive-default, + // the curated marketplace bundles (cncf-techdocs-assessment), the ingest + // skeleton (Intake, Triage) and the editor palette has an entry; the + // workflow editor's column inspector shows the one its label resolves to. + const STAGE_ICON_BY_NAME = { + // hive-default + 'open': 'inbox', + 'actionable': 'bolt', + 'claimed': 'hand', + 'pr in progress': 'pull-request', + 'ci review': 'flask', + 'ci': 'flask', + 'review': 'eye', + 'merge queue': 'merge', + 'merged': 'check', + // cncf-techdocs-assessment + 'requested': 'inbox', + 'accepted': 'checkbox', + 'drafting': 'pencil', + 'reviewer check': 'eye', + 'stakeholder review': 'people', + 'approval': 'thumbs-up', + 'done': 'check', + // ingest skeleton and editor palette + 'intake': 'inbox', + 'triage': 'tag', + 'analysis': 'search', + 'writing': 'pencil', + 'publish': 'rocket', + 'published': 'rocket', + 'needs decision': 'person', + 'work in progress': 'pencil', + 'in progress': 'pencil', + 'in review': 'eye', + 'ci failing': 'flask', + 'released': 'rocket', + 'deployed': 'rocket', + 'deployed staging': 'rocket', + 'deployed production': 'rocket', + 'closed': 'archive', + 'verified': 'shield-check', + 'verification': 'shield-check', + 'approved': 'thumbs-up', + 'blocked': 'pause', + 'hold': 'pause', + 'on hold': 'pause', + 'waiting': 'clock', + 'backlog': 'list', + 'todo': 'list', + 'to do': 'list', + 'docs': 'book', + 'documentation': 'book', + }; + + // Pure: is v an allowlisted icon id? + function stageIconIsId(v) { return Object.prototype.hasOwnProperty.call(STAGE_ICON_SET, String(v || '')); } + + // Pure: is v a single emoji — one or two pictographs (a ZWJ sequence or + // a flag) plus modifiers, no letters, digits, spaces or markup? The + // mirror of workflowmodel.IsColumnIconEmoji. + function stageIconIsEmoji(v) { + const s = String(v == null ? '' : v); + if (!s || Array.from(s).length > 8) return false; + let pict = 0; + for (const ch of s) { + const cp = ch.codePointAt(0); + if (cp === 0x200D || cp === 0xFE0F || cp === 0xFE0E || cp === 0x20E3 || (cp >= 0x1F3FB && cp <= 0x1F3FF)) continue; + if (cp < 0x80 || /[\p{L}\p{N}\s\p{Cc}]/u.test(ch)) return false; + if ((cp >= 0x1F000 && cp <= 0x1FAFF) || (cp >= 0x2600 && cp <= 0x27BF) || (cp >= 0x2B00 && cp <= 0x2BFF) || (cp >= 0x2300 && cp <= 0x23FF) || (cp >= 0x2190 && cp <= 0x21FF) || (cp >= 0x25A0 && cp <= 0x25FF) || (cp >= 0x2900 && cp <= 0x297F) || (cp >= 0x1F1E6 && cp <= 0x1F1FF) || [0xA9, 0xAE, 0x2122, 0x2139, 0x3030, 0x303D, 0x3297, 0x3299].indexOf(cp) >= 0) { pict++; continue; } + return false; + } + return pict >= 1 && pict <= 2; + } + + // Pure: '' (default), an icon id or a single emoji validate; anything + // else is refused — the editor shows why and keeps the last good value. + function stageIconIsValid(v) { + const s = String(v == null ? '' : v).trim(); + return s === '' || stageIconIsId(s) || stageIconIsEmoji(s); + } + + // Pure: the icon id a column name resolves to; STAGE_ICON_DEFAULT when + // the name is unknown. + function stageIconIdForName(name) { + return STAGE_ICON_BY_NAME[stageIconNormalizeName(name)] || STAGE_ICON_DEFAULT; + } + + // Pure: what a column draws — its valid `icon` override (id or emoji), + // else the id its label resolves to. An invalid override is ignored, + // never rendered. + function stageIconFor(col) { + const c = col || {}; + const over = String(c.icon == null ? '' : c.icon).trim(); + if (over && (stageIconIsId(over) || stageIconIsEmoji(over))) return over; + return stageIconIdForName(c.label || c.id); + } + + // The icon as markup: an allowlisted id becomes its inline SVG, an + // emoji a text span (escaped), anything else nothing. label is for + // assistive tech only; the chevron's title carries the hover text. + function stageIconHtml(icon, cls, label) { + const v = String(icon == null ? '' : icon).trim(); + const aria = label ? ` role="img" aria-label="${esc(label)}"` : ' aria-hidden="true"'; + if (stageIconIsId(v)) return `<svg class="${cls}" viewBox="0 0 16 16" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"${aria}>${STAGE_ICON_SET[v]}</svg>`; + if (stageIconIsEmoji(v)) return `<span class="${cls} ham-emoji-ico"${aria}>${esc(v)}</span>`; + return ''; + } + + // The glyph a column's chevron wears, by its override or its name. + function stageIconColumnHtml(col, cls, label) { + return stageIconHtml(stageIconFor(col), cls, label); + } diff --git a/src/pkg/dashboard/static-src/js/tracker-workflows.js b/src/pkg/dashboard/static-src/js/tracker-workflows.js index fa24eacf3d..5004eac845 100644 --- a/src/pkg/dashboard/static-src/js/tracker-workflows.js +++ b/src/pkg/dashboard/static-src/js/tracker-workflows.js @@ -67,7 +67,7 @@ // The model's column header, drawn under a bound repository's group // header in place of the shared hive-default header. function stageTrackerWorkflowHeaderHtml(wf) { - const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${esc(c.label)}</span></li>`).join(''); + const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-col-ico ham-stage-ico')}${esc(c.label)}</span></li>`).join(''); return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } @@ -127,10 +127,20 @@ return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; } + // The glyph a model column wears (hivecommons/hive#18033): its `icon` + // override when valid, else the icon its label resolves to. Guarded so + // harnesses that lift the strip alone still render. + function stageTrackerWorkflowColumnIconHtml(c, cls, label) { + return typeof stageIconColumnHtml === 'function' ? stageIconColumnHtml(c, cls, label) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber // with the "Not covered" word so the gap is visible, not silent. + // Done and todo chevrons wear the column's icon alone (the label is + // the header's and the hover's; in a chevron it only clipped); the + // active one leads its word with the icon (hivecommons/hive#18033). function stageTrackerWorkflowStripItems(item, wf, stages) { const cur = Number(item.stage) || 0; const terminal = cur >= (typeof STAGE_TRACKER_TERMINAL === 'number' ? STAGE_TRACKER_TERMINAL : 110); @@ -148,6 +158,8 @@ ? `stage/${String(cur).padStart(3, '0')} is not a state of ${wf.name}@${wf.version}; the engine leaves this item where it is.` : (c.hint || c.label) + (state === 'active' && cur ? ` — stage/${String(cur).padStart(3, '0')}` : ''); const cls = ['ham-seg', 'ham-wf-seg', state, c.terminal ? 'cap' : '', uncovered && state === 'active' ? 'ham-wf-uncovered' : ''].filter(Boolean).join(' '); - return `<li class="${cls}" title="${esc(tip)}"><span class="ham-seg-t">${esc(word)}</span></li>`; + const ico = stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-stage-ico'); + const inner = state === 'active' ? `${ico}<span class="ham-seg-t">${esc(word)}</span>` : (ico || `<span class="ham-seg-t">${esc(word)}</span>`); + return `<li class="${cls}" title="${esc(tip)}">${inner}</li>`; }).join(''); } diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index fe1b1cd620..310a469a9b 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -81,6 +81,7 @@ js/tracker-words-gate.js js/tracker-words.js js/tracker-row-anchor.js js/tracker-row.js +js/tracker-stage-icons.js js/tracker-row-strip.js js/tracker-workflows.js js/tracker-row-acted.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 342530dd61..a59be35a8e 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -1615,6 +1615,14 @@ .ham-wf-badge { flex: 0 0 auto; font-family: var(--font-mono); font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; color: var(--indigo); background: color-mix(in srgb, var(--indigo) 12%, transparent); border: 1px solid color-mix(in srgb, var(--indigo) 40%, transparent); } .ham-wf-cols { position: static; } .ham-wf-cols .ham-wf-col { color: var(--indigo); } + /* Model chevrons wear the column's stage icon (hivecommons/hive issue 18033): + alone and dimmed in a done / todo step, leading the word in the lit + one. An emoji override is sized to the SVG glyph so the row keeps + its height. */ + .ham-seg.ham-wf-seg .ham-stage-ico { flex: 0 0 auto; } + .ham-seg.ham-wf-seg.active .ham-stage-ico { margin-right: var(--sp-1); } + .ham-seg.ham-wf-seg.done .ham-stage-ico { opacity: 0.6; } + .ham-emoji-ico { display: inline-block; width: 12px; height: 12px; font-size: var(--fs-xs); line-height: 12px; text-align: center; font-style: normal; } .ham-seg.ham-wf-uncovered { background: color-mix(in srgb, var(--amber) 18%, transparent); color: var(--amber); outline: 1px dashed var(--amber); } .ham-group-dist { flex: 0 0 auto; margin-left: auto; margin-right: calc(4em + var(--sp-3)); display: inline-flex; align-items: flex-end; gap: var(--sp-1); height: 0.9em; } .ham-gd { display: inline-block; width: 0.45em; height: 15%; border-radius: var(--sp-1); background: color-mix(in srgb, var(--text-faint) 35%, transparent); } @@ -5298,6 +5306,12 @@ .wf-ed-field input[readonly] { color: var(--muted); } .wf-ed-inline { display: flex; gap: var(--sp-2); align-items: center; } .wf-ed-inline input { flex: 1 1 auto; min-width: 0; } + /* Column icon row (hivecommons/hive#18033): preview glyph, id pick-list, emoji box. */ + .wf-ed-icon-row .wf-ed-icon-preview { display: inline-flex; align-items: center; justify-content: center; width: 24px; height: 24px; flex: 0 0 auto; border-radius: var(--r-sm, 4px); background: var(--panel-strong); color: var(--text); } + .wf-ed-icon-row .wf-ed-icon-preview .ham-ico { width: 14px; height: 14px; } + .wf-ed-icon-row .wf-ed-icon-preview .ham-emoji-ico { width: 14px; height: 14px; font-size: var(--fs-sm); line-height: 14px; } + .wf-ed-icon-row select { flex: 1 1 auto; min-width: 0; } + .wf-ed-icon-row .wf-ed-icon-emoji { flex: 0 0 7em; } .wf-ed-checks { display: flex; gap: var(--sp-4); align-items: center; flex-wrap: wrap; } .wf-ed-warn { color: var(--status-warn); } .wf-ed-translist { margin: var(--sp-0); padding-left: var(--sp-5); font-size: var(--fs-xs); } @@ -18803,6 +18817,166 @@ <h3 class="faq-group-title">Where to get help</h3> if (hours < 48) return hours + 'h'; return Math.floor(hours / 24) + 'd'; } +/* ---- Stage icons for workflow-model chevrons (hivecommons/hive#18033) --- + A chevron that is not the row's active stage used to carry only its + clipped column label. Now every column has a glyph that says what the + stage means: picked by the column's name (STAGE_ICON_BY_NAME, keyed + by the normalised label), or by the model's own `icon` on the column + — an allowlisted id below or a single emoji, never markup. The set is + 16-unit monochrome strokes in currentColor, so it follows the theme + like STAGE_TRACKER_ICONS does. Pure helpers: the node harness + (stage_tracker_stage_icons_ui_test.go) lifts them by name, and a Go + test keeps the id list equal to workflowmodel.ColumnIconIDs. */ + const STAGE_ICON_SET = { + inbox: '<path d="M2 9.5h3.5l1 2h3l1-2H14"/><path d="M2 9.5 4 3.5h8l2 6v4H2z"/>', + circle: '<circle cx="8" cy="8" r="5"/>', + bolt: '<path d="M9 1 3 9h4l-1 6 6-8H8z"/>', + hand: '<path d="M4.5 8.5V4a1 1 0 0 1 2 0v3.5M6.5 7V3a1 1 0 0 1 2 0v4M8.5 7V3.5a1 1 0 0 1 2 0V8M10.5 8V5a1 1 0 0 1 2 0v5.5c0 2.5-1.8 4-4.3 4-2 0-3-.9-3.9-2.3L2.6 9.9a1 1 0 0 1 1.6-1.2L4.5 9"/>', + person: '<circle cx="8" cy="5" r="3"/><path d="M2.5 14.5a5.5 5.5 0 0 1 11 0"/>', + pencil: '<path d="M11.5 2.5 13.5 4.5 5.5 12.5 2.5 13.5 3.5 10.5z"/><path d="M9.5 4.5l2 2"/>', + 'pull-request': '<circle cx="4" cy="3" r="1.5"/><circle cx="4" cy="13" r="1.5"/><circle cx="12" cy="13" r="1.5"/><path d="M4 4.5v7M12 11.5V7a2 2 0 0 0-2-2H8"/>', + flask: '<path d="M5.5 1.5h5M6.5 1.5v4.6L2.6 12.9a1 1 0 0 0 .9 1.6h9a1 1 0 0 0 .9-1.6L9.5 6.1V1.5"/><path d="M4.4 10.5h7.2"/>', + gear: '<circle cx="8" cy="8" r="2.2"/><path d="M8 1.5v2M8 12.5v2M1.5 8h2M12.5 8h2M3.4 3.4l1.4 1.4M11.2 11.2l1.4 1.4M3.4 12.6l1.4-1.4M11.2 4.8l1.4-1.4"/>', + eye: '<path d="M1 8s2.5-5 7-5 7 5 7 5-2.5 5-7 5-7-5-7-5z"/><circle cx="8" cy="8" r="2"/>', + people: '<circle cx="5.5" cy="5" r="2.5"/><path d="M1 14a4.5 4.5 0 0 1 9 0"/><path d="M10.5 3a2.5 2.5 0 0 1 0 4.5M12 10a4 4 0 0 1 3 4"/>', + 'thumbs-up': '<path d="M5 7.5 8 1.5c1.5 0 2 1 2 2L9.5 6H13a1.5 1.5 0 0 1 1.5 1.7l-1 5A1.5 1.5 0 0 1 12 14H5z"/><path d="M5 7.5H2v6.5h3"/>', + 'shield-check': '<path d="M8 1.5 13.5 3.5v4c0 3.5-2.5 5.9-5.5 7-3-1.1-5.5-3.5-5.5-7v-4z"/><path d="M5.5 8l1.8 1.8L10.8 6"/>', + merge: '<path stroke-width="2" d="M6 14.5V3M3 6l3-3 3 3M13 14.5c0-3.8-3.4-4.6-7-6.5"/>', + check: '<path d="M3 8l3 3 7-7"/>', + checkbox: '<rect x="2.5" y="2.5" width="11" height="11" rx="1.5"/><path d="M5 8l2 2 4-4"/>', + list: '<path d="M3 5h10M3 8h10M3 11h10"/>', + tag: '<path d="M2.5 2.5h5.5l5.5 5.5-5.5 5.5-5.5-5.5z"/><circle cx="5.5" cy="5.5" r="1"/>', + flag: '<path d="M3.5 14.5V2.5h8.5l-2 3 2 3H3.5"/>', + rocket: '<path d="M8 1.5c2.5 1.5 3.5 4.5 3 8l-3 3-3-3c-.5-3.5.5-6.5 3-8z"/><circle cx="8" cy="6.5" r="1.2"/><path d="M5 9.5 3 11.5l.5 2 2-1M11 9.5l2 2-.5 2-2-1"/>', + book: '<path d="M2.5 2.5h4a1.5 1.5 0 0 1 1.5 1.5v9.5a1 1 0 0 0-1-1h-4.5zM13.5 2.5h-4A1.5 1.5 0 0 0 8 4v9.5a1 1 0 0 1 1-1h4.5z"/>', + search: '<circle cx="7" cy="7" r="4.5"/><path d="M10.5 10.5 14 14"/>', + clock: '<circle cx="8" cy="8" r="6"/><path d="M8 4.5V8l2.5 1.5"/>', + pause: '<path d="M5.5 3v10M10.5 3v10"/>', + archive: '<rect x="2" y="2.5" width="12" height="3"/><path d="M3 5.5v8h10v-8M6.5 8.5h3"/>', + dot: '<circle cx="8" cy="8" r="1.75" fill="currentColor"/>', + }; + const STAGE_ICON_DEFAULT = 'dot'; + + // Pure: the key a column name is matched by — lower-case, trimmed, + // inner whitespace and the " / " and " · " separators collapsed to one + // space, so "Reviewer check", "reviewer check" and "CI / Review" each + // land on one entry. + function stageIconNormalizeName(name) { + return String(name == null ? '' : name).toLowerCase().replace(/[\s·/]+/g, ' ').trim(); + } + + // Canonical stage name → icon id. Every column label across hive-default, + // the curated marketplace bundles (cncf-techdocs-assessment), the ingest + // skeleton (Intake, Triage) and the editor palette has an entry; the + // workflow editor's column inspector shows the one its label resolves to. + const STAGE_ICON_BY_NAME = { + // hive-default + 'open': 'inbox', + 'actionable': 'bolt', + 'claimed': 'hand', + 'pr in progress': 'pull-request', + 'ci review': 'flask', + 'ci': 'flask', + 'review': 'eye', + 'merge queue': 'merge', + 'merged': 'check', + // cncf-techdocs-assessment + 'requested': 'inbox', + 'accepted': 'checkbox', + 'drafting': 'pencil', + 'reviewer check': 'eye', + 'stakeholder review': 'people', + 'approval': 'thumbs-up', + 'done': 'check', + // ingest skeleton and editor palette + 'intake': 'inbox', + 'triage': 'tag', + 'analysis': 'search', + 'writing': 'pencil', + 'publish': 'rocket', + 'published': 'rocket', + 'needs decision': 'person', + 'work in progress': 'pencil', + 'in progress': 'pencil', + 'in review': 'eye', + 'ci failing': 'flask', + 'released': 'rocket', + 'deployed': 'rocket', + 'deployed staging': 'rocket', + 'deployed production': 'rocket', + 'closed': 'archive', + 'verified': 'shield-check', + 'verification': 'shield-check', + 'approved': 'thumbs-up', + 'blocked': 'pause', + 'hold': 'pause', + 'on hold': 'pause', + 'waiting': 'clock', + 'backlog': 'list', + 'todo': 'list', + 'to do': 'list', + 'docs': 'book', + 'documentation': 'book', + }; + + // Pure: is v an allowlisted icon id? + function stageIconIsId(v) { return Object.prototype.hasOwnProperty.call(STAGE_ICON_SET, String(v || '')); } + + // Pure: is v a single emoji — one or two pictographs (a ZWJ sequence or + // a flag) plus modifiers, no letters, digits, spaces or markup? The + // mirror of workflowmodel.IsColumnIconEmoji. + function stageIconIsEmoji(v) { + const s = String(v == null ? '' : v); + if (!s || Array.from(s).length > 8) return false; + let pict = 0; + for (const ch of s) { + const cp = ch.codePointAt(0); + if (cp === 0x200D || cp === 0xFE0F || cp === 0xFE0E || cp === 0x20E3 || (cp >= 0x1F3FB && cp <= 0x1F3FF)) continue; + if (cp < 0x80 || /[\p{L}\p{N}\s\p{Cc}]/u.test(ch)) return false; + if ((cp >= 0x1F000 && cp <= 0x1FAFF) || (cp >= 0x2600 && cp <= 0x27BF) || (cp >= 0x2B00 && cp <= 0x2BFF) || (cp >= 0x2300 && cp <= 0x23FF) || (cp >= 0x2190 && cp <= 0x21FF) || (cp >= 0x25A0 && cp <= 0x25FF) || (cp >= 0x2900 && cp <= 0x297F) || (cp >= 0x1F1E6 && cp <= 0x1F1FF) || [0xA9, 0xAE, 0x2122, 0x2139, 0x3030, 0x303D, 0x3297, 0x3299].indexOf(cp) >= 0) { pict++; continue; } + return false; + } + return pict >= 1 && pict <= 2; + } + + // Pure: '' (default), an icon id or a single emoji validate; anything + // else is refused — the editor shows why and keeps the last good value. + function stageIconIsValid(v) { + const s = String(v == null ? '' : v).trim(); + return s === '' || stageIconIsId(s) || stageIconIsEmoji(s); + } + + // Pure: the icon id a column name resolves to; STAGE_ICON_DEFAULT when + // the name is unknown. + function stageIconIdForName(name) { + return STAGE_ICON_BY_NAME[stageIconNormalizeName(name)] || STAGE_ICON_DEFAULT; + } + + // Pure: what a column draws — its valid `icon` override (id or emoji), + // else the id its label resolves to. An invalid override is ignored, + // never rendered. + function stageIconFor(col) { + const c = col || {}; + const over = String(c.icon == null ? '' : c.icon).trim(); + if (over && (stageIconIsId(over) || stageIconIsEmoji(over))) return over; + return stageIconIdForName(c.label || c.id); + } + + // The icon as markup: an allowlisted id becomes its inline SVG, an + // emoji a text span (escaped), anything else nothing. label is for + // assistive tech only; the chevron's title carries the hover text. + function stageIconHtml(icon, cls, label) { + const v = String(icon == null ? '' : icon).trim(); + const aria = label ? ` role="img" aria-label="${esc(label)}"` : ' aria-hidden="true"'; + if (stageIconIsId(v)) return `<svg class="${cls}" viewBox="0 0 16 16" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"${aria}>${STAGE_ICON_SET[v]}</svg>`; + if (stageIconIsEmoji(v)) return `<span class="${cls} ham-emoji-ico"${aria}>${esc(v)}</span>`; + return ''; + } + + // The glyph a column's chevron wears, by its override or its name. + function stageIconColumnHtml(col, cls, label) { + return stageIconHtml(stageIconFor(col), cls, label); + } // Column header: the stage names once, aligned with the chevrons below. // It is sticky under the Merged bar (tracker-cues CSS), so the names @@ -19966,7 +20140,7 @@ <h3 class="faq-group-title">Where to get help</h3> // The model's column header, drawn under a bound repository's group // header in place of the shared hive-default header. function stageTrackerWorkflowHeaderHtml(wf) { - const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${esc(c.label)}</span></li>`).join(''); + const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-col-ico ham-stage-ico')}${esc(c.label)}</span></li>`).join(''); return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } @@ -20026,10 +20200,20 @@ <h3 class="faq-group-title">Where to get help</h3> return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; } + // The glyph a model column wears (hivecommons/hive#18033): its `icon` + // override when valid, else the icon its label resolves to. Guarded so + // harnesses that lift the strip alone still render. + function stageTrackerWorkflowColumnIconHtml(c, cls, label) { + return typeof stageIconColumnHtml === 'function' ? stageIconColumnHtml(c, cls, label) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber // with the "Not covered" word so the gap is visible, not silent. + // Done and todo chevrons wear the column's icon alone (the label is + // the header's and the hover's; in a chevron it only clipped); the + // active one leads its word with the icon (hivecommons/hive#18033). function stageTrackerWorkflowStripItems(item, wf, stages) { const cur = Number(item.stage) || 0; const terminal = cur >= (typeof STAGE_TRACKER_TERMINAL === 'number' ? STAGE_TRACKER_TERMINAL : 110); @@ -20047,7 +20231,9 @@ <h3 class="faq-group-title">Where to get help</h3> ? `stage/${String(cur).padStart(3, '0')} is not a state of ${wf.name}@${wf.version}; the engine leaves this item where it is.` : (c.hint || c.label) + (state === 'active' && cur ? ` — stage/${String(cur).padStart(3, '0')}` : ''); const cls = ['ham-seg', 'ham-wf-seg', state, c.terminal ? 'cap' : '', uncovered && state === 'active' ? 'ham-wf-uncovered' : ''].filter(Boolean).join(' '); - return `<li class="${cls}" title="${esc(tip)}"><span class="ham-seg-t">${esc(word)}</span></li>`; + const ico = stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-stage-ico'); + const inner = state === 'active' ? `${ico}<span class="ham-seg-t">${esc(word)}</span>` : (ico || `<span class="ham-seg-t">${esc(word)}</span>`); + return `<li class="${cls}" title="${esc(tip)}">${inner}</li>`; }).join(''); } const STAGE_TRACKER_ACTED_STATES = ['waiting', 'addressed', 'stuck', 'pending', 'failed', 'executing', 'blocked']; @@ -65606,6 +65792,24 @@ <h4 style="margin:18px 0 8px">Agent Watchdog</h4> options.map(o => { const v = Array.isArray(o) ? o[0] : o, l = Array.isArray(o) ? o[1] : (o || '\u2014'); return `<option value="${esc(v)}" ${v === value ? 'selected' : ''}>${esc(l)}</option>`; }).join('') + '</select>'; } + // The column's chevron icon (hivecommons/hive#18033): a preview of what + // the Activity Monitor draws, a pick-list of the allowlisted ids with + // the label's default first, and a box for a single emoji. Both write + // spec.columns[id].icon through wfEdSetColumnIcon, which refuses + // anything outside the allowlist. + function wfEdColumnIconField(c, base, dis) { + if (typeof stageIconFor !== 'function') return ''; + const cur = String(c.icon == null ? '' : c.icon).trim(); + const def = stageIconIdForName(c.label || c.id); + const isEmoji = !!cur && !stageIconIsId(cur); + const ids = Object.keys(STAGE_ICON_SET); + const options = [['', 'default \u00b7 ' + def]].concat(ids.map(id => [id, id])); + const preview = `<span class="wf-ed-icon-preview" title="${esc('as drawn in the Activity Monitor')}">${stageIconHtml(stageIconFor(c), 'ham-ico ham-stage-ico')}</span>`; + const select = `<select data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>${options.map(o => `<option value="${esc(o[0])}" ${!isEmoji && o[0] === cur ? 'selected' : ''}>${esc(o[1])}</option>`).join('')}${isEmoji ? `<option value="${esc(cur)}" selected>${esc(cur)} (emoji)</option>` : ''}</select>`; + const emoji = `<input type="text" class="wf-ed-icon-emoji" maxlength="16" placeholder="or an emoji" value="${esc(isEmoji ? cur : '')}" data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>`; + return wfEdField('Icon', `<div class="wf-ed-inline wf-ed-icon-row">${preview}${select}${emoji}</div>`, 'drawn in this column\u2019s chevrons; blank = by label'); + } + function renderWfEdInspector(st) { const m = st.model, spec = m.spec, sel = st.selection; const can = wfEdCanEdit(); @@ -65656,6 +65860,7 @@ <h4 style="margin:18px 0 8px">Agent Watchdog</h4> wfEdField('Id', `<div class="wf-ed-inline"><input type="text" value="${esc(c.id)}" readonly><button type="button" class="hv-btn btn-secondary btn-sm" data-action="wfEdRenameColumnAction" data-arg0="${esc(c.id)}" ${dis}>Rename\u2026</button></div>`) + wfEdField('Label', wfEdText(base + '.label', c.label, dis)) + wfEdField('Hint', wfEdText(base + '.hint', c.hint, dis), 'shown as the lane\u2019s tooltip in the Activity Monitor') + + wfEdColumnIconField(c, base, dis) + `<div class="config-field wf-ed-field"><label><input type="checkbox" ${c.terminal ? 'checked' : ''} data-change-action="wfEdFieldSetBool" data-arg0="${esc(base + '.terminal')}" data-arg-types="s,c" ${dis}> Terminal column</label><span class="text-muted fs-xs">states here need no exit</span></div>` + wfEdField('Width', `<input type="range" min="${WF_ED.minColW}" max="${WF_ED.maxColW}" step="${WF_ED.grid}" value="${st.view.laneWidths[c.id] || WF_ED.colW}" data-input-action="wfEdLaneWidth" data-arg0="${esc(c.id)}" data-arg-types="s,v">`) + `<p class="text-muted fs-sm">${(c.states || []).length} state${(c.states || []).length === 1 ? '' : 's'}</p>` + @@ -66227,6 +66432,19 @@ <h4 style="margin:18px 0 8px">Agent Watchdog</h4> wfEdCommit(wfEdUpdate(_wfEd.model, path, isNaN(n) || n <= 0 ? '' : n)); } function wfEdFieldSetBool(path, checked) { wfEdCommit(wfEdUpdate(_wfEd.model, path, checked ? true : '')); } + // An icon is '' (by label), an allowlisted id or one emoji; anything + // else is refused with a toast and the inspector redraws the last good + // value, so no markup ever reaches a chevron (hivecommons/hive#18033). + function wfEdSetColumnIcon(path, value) { + if (!_wfEd.model) return; + const v = String(value == null ? '' : value).trim(); + if (typeof stageIconIsValid === 'function' && !stageIconIsValid(v)) { + if (typeof showToast === 'function') showToast('Icon must be one of the built-in icon ids or a single emoji.', 'error'); + wfEdRefresh(['inspector']); + return; + } + wfEdCommit(wfEdUpdate(_wfEd.model, path, v), { parts: ['canvas', 'inspector', 'drawer'] }); + } function wfEdSetList(path, value) { wfEdCommit(wfEdUpdate(_wfEd.model, path, String(value || '').split(/[,\s]+/).map(s => s.trim()).filter(Boolean))); } function wfEdSetStateColumn(id, columnId) { wfEdCommit(wfEdMoveState(_wfEd.model, id, columnId === WF_ED_UNASSIGNED ? '' : columnId)); } function wfEdSetInitial(id) { wfEdCommit(wfEdUpdate(_wfEd.model, 'spec.initial', id)); } diff --git a/src/pkg/workflowmodel/icons.go b/src/pkg/workflowmodel/icons.go new file mode 100644 index 0000000000..20cae6bcd4 --- /dev/null +++ b/src/pkg/workflowmodel/icons.go @@ -0,0 +1,97 @@ +package workflowmodel + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +// ColumnIconIDs is the allowlist of icon ids a Column.Icon may name. The +// SPA draws each id from its own inline-SVG set (STAGE_ICON_SET in +// static-src/js/tracker-stage-icons.js; a dashboard test keeps the two +// lists equal), so a model can never inject markup into a chevron +// (hivecommons/hive#18033). Order is the editor's pick-list order. +var ColumnIconIDs = []string{ + "inbox", "circle", "bolt", "hand", "person", "pencil", "pull-request", + "flask", "gear", "eye", "people", "thumbs-up", "shield-check", "merge", + "check", "checkbox", "list", "tag", "flag", "rocket", "book", "search", + "clock", "pause", "archive", "dot", +} + +// ColumnIconDefault is the icon an unknown column name falls back to. +const ColumnIconDefault = "dot" + +// columnIconMaxEmojiRunes bounds an emoji override: one pictograph plus +// its variation selector, skin tone or ZWJ-joined companions. +const columnIconMaxEmojiRunes = 8 + +var columnIconIDSet = func() map[string]bool { + m := make(map[string]bool, len(ColumnIconIDs)) + for _, id := range ColumnIconIDs { + m[id] = true + } + return m +}() + +// IsColumnIconID reports whether id is an allowlisted icon id. +func IsColumnIconID(id string) bool { return columnIconIDSet[id] } + +// IsColumnIconEmoji reports whether s is a single emoji (one pictograph +// with optional modifiers and joiners): no letters, digits, spaces, +// punctuation or markup, so a user-typed override can only ever render as +// a glyph. +func IsColumnIconEmoji(s string) bool { + if s == "" || !utf8.ValidString(s) || utf8.RuneCountInString(s) > columnIconMaxEmojiRunes { + return false + } + pictographs := 0 + for _, r := range s { + switch { + case columnIconJoiner(r): + case columnIconPictograph(r): + pictographs++ + default: + return false + } + } + return pictographs >= 1 && pictographs <= 2 +} + +// columnIconJoiner: ZWJ, variation selectors, keycap and skin tones — the +// code points that modify a pictograph without being one. +func columnIconJoiner(r rune) bool { + return r == 0x200D || r == 0xFE0F || r == 0xFE0E || r == 0x20E3 || (r >= 0x1F3FB && r <= 0x1F3FF) +} + +// columnIconPictograph covers the emoji blocks (Supplemental Symbols and +// Pictographs, Misc Symbols, Dingbats, Arrows, Geometric Shapes, …) and +// the regional indicators that pair into flags. +func columnIconPictograph(r rune) bool { + if unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsSpace(r) || unicode.IsControl(r) || r < 0x80 { + return false + } + ranges := [][2]rune{{0x1F000, 0x1FAFF}, {0x2600, 0x27BF}, {0x2B00, 0x2BFF}, {0x2300, 0x23FF}, {0x2190, 0x21FF}, {0x25A0, 0x25FF}, {0x2900, 0x297F}, {0x1F1E6, 0x1F1FF}} + for _, rg := range ranges { + if r >= rg[0] && r <= rg[1] { + return true + } + } + switch r { + case 0x00A9, 0x00AE, 0x2122, 0x2139, 0x3030, 0x303D, 0x3297, 0x3299: + return true + } + return false +} + +// ValidateColumnIcon accepts the empty value (default by name), an +// allowlisted icon id, or a single emoji; anything else is an error naming +// what was rejected. Only surrounding blanks are forgiven: a newline or +// other control character is never part of an icon. +func ValidateColumnIcon(icon string) error { + v := strings.Trim(icon, " \t") + if v == "" || IsColumnIconID(v) || IsColumnIconEmoji(v) { + return nil + } + return fmt.Errorf("icon %q is neither an icon id (%s) nor a single emoji", icon, strings.Join(ColumnIconIDs, ", ")) +} diff --git a/src/pkg/workflowmodel/icons_test.go b/src/pkg/workflowmodel/icons_test.go new file mode 100644 index 0000000000..60d8f8ff66 --- /dev/null +++ b/src/pkg/workflowmodel/icons_test.go @@ -0,0 +1,60 @@ +package workflowmodel + +import ( + "errors" + "strings" + "testing" +) + +// A column's icon is the empty default, an allowlisted id or a single +// emoji; letters, markup and multi-glyph strings are refused so the SPA's +// chevron renderer never sees anything it would have to escape +// (hivecommons/hive#18033). +func TestValidateColumnIcon(t *testing.T) { + for _, ok := range []string{"", "bolt", "merge", " eye ", "✅", "🚀", "👍🏽", "👩‍💻", "🇺🇸", "⏸️"} { + if err := ValidateColumnIcon(ok); err != nil { + t.Errorf("ValidateColumnIcon(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{"sparkle", "Bolt", "<svg>", "<img src=x onerror=alert(1)>", "a", "✅✅✅", "ok ✅", "🚀 launch", "javascript:", "✅\n"} { + if err := ValidateColumnIcon(bad); err == nil { + t.Errorf("ValidateColumnIcon(%q) = nil, want error", bad) + } + } +} + +func TestValidateRejectsBadColumnIcon(t *testing.T) { + m := techdocs(t) + m.Spec.Columns[0].Icon = "<b>x</b>" + m.Spec.Columns[1].Icon = "bolt" + err := m.Validate() + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v", err) + } + if !strings.Contains(err.Error(), `column "`+m.Spec.Columns[0].ID+`": icon "<b>x</b>"`) { + t.Errorf("error does not name the column and icon: %v", err) + } + if strings.Contains(err.Error(), `"bolt"`) { + t.Errorf("allowlisted id reported: %v", err) + } + m.Spec.Columns[0].Icon = "📥" + if err := m.Validate(); err != nil { + t.Errorf("emoji icon rejected: %v", err) + } +} + +func TestColumnIconIDsAreUnique(t *testing.T) { + seen := map[string]bool{} + for _, id := range ColumnIconIDs { + if seen[id] { + t.Errorf("duplicate icon id %q", id) + } + seen[id] = true + if strings.TrimSpace(id) != id || strings.ToLower(id) != id { + t.Errorf("icon id %q is not a trimmed lower-case token", id) + } + } + if !IsColumnIconID(ColumnIconDefault) { + t.Errorf("default %q is not in the allowlist", ColumnIconDefault) + } +} diff --git a/src/pkg/workflowmodel/lifecycle.go b/src/pkg/workflowmodel/lifecycle.go index 6606df5364..ef3c1db93d 100644 --- a/src/pkg/workflowmodel/lifecycle.go +++ b/src/pkg/workflowmodel/lifecycle.go @@ -64,9 +64,13 @@ type Citation struct { // Column is one Activity Monitor column; several states may share it. type Column struct { - ID string `json:"id"` - Label string `json:"label"` - Hint string `json:"hint,omitempty"` + ID string `json:"id"` + Label string `json:"label"` + Hint string `json:"hint,omitempty"` + // Icon overrides the glyph the Activity Monitor draws in this column's + // chevrons: an allowlisted id from ColumnIconIDs or a single emoji; + // empty picks the icon by the column's label (hivecommons/hive#18033). + Icon string `json:"icon,omitempty"` States []string `json:"states"` Terminal bool `json:"terminal,omitempty"` } diff --git a/src/pkg/workflowmodel/model.go b/src/pkg/workflowmodel/model.go index 7ec6c90386..4d30b96249 100644 --- a/src/pkg/workflowmodel/model.go +++ b/src/pkg/workflowmodel/model.go @@ -341,6 +341,11 @@ func (m *Model) Validate() error { if m.Spec.Scope.Repository == "" { fail("spec.scope.repository is empty") } + for _, c := range m.Spec.Columns { + if err := ValidateColumnIcon(c.Icon); err != nil { + fail("column %q: %v", c.ID, err) + } + } ids := map[string]bool{} var walk func(states []State, parallelParent bool) walk = func(states []State, parallelParent bool) { From 874763ceccf63104ed21cd576f3494b6872969de Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:16:50 +0000 Subject: [PATCH 138/223] [scanner] fix: dedupe expected agent role claim rejection log (#18076) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- changelog.d/fixed-role-claim-reject-log.md | 1 + src/pkg/dashboard/contribute_ws.go | 27 ++++++++++++++++--- .../contribute_ws_role_reject_log_test.go | 25 +++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-role-claim-reject-log.md create mode 100644 src/pkg/dashboard/contribute_ws_role_reject_log_test.go diff --git a/changelog.d/fixed-role-claim-reject-log.md b/changelog.d/fixed-role-claim-reject-log.md new file mode 100644 index 0000000000..e5ca0de6d8 --- /dev/null +++ b/changelog.d/fixed-role-claim-reject-log.md @@ -0,0 +1 @@ +- Repeated trust-tier agent role claim rejections over contribute-ws are now logged once per user and role at INFO instead of flooding WARN, so they no longer surface as unclassified self-reported errors (#18071). diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index f9c4e86eed..4023c44b25 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -1089,6 +1089,9 @@ type ContributeWSHub struct { // stays bounded. Guarded by rateMu. assignmentTimes map[string][]time.Time rateMu sync.Mutex + // roleRejectLogged dedupes the "agent role claim rejected" log per + // (user, role) so a client retrying a forbidden role does not flood it. + roleRejectLogged sync.Map // map[string]time.Time // contributorFailureStreaks tracks, per identity, CONSECUTIVE hub-measured // sub-minute task failures (kubestellar/hive#6450, contribute_failure_streak.go). // Once a streak reaches contributorFailureStreakThreshold, selectTask pauses @@ -2843,9 +2846,7 @@ func (s *wsSession) handleAuthResponse(msg WSMessage) (stop bool) { if requestedRole != "" { if ok, reason := h.roleClaimAllowed(probeContributor, requestedRole); !ok { _ = sendJSON(s.conn, WSMessage{Type: "auth_failed", Reason: reason, Role: requestedRole}) - h.logger.Warn("[contribute-ws] agent role claim rejected", - "username", profile.GitHubUsername, "tier", profile.TrustTier, - "role", requestedRole, "reason", reason) + h.logRoleClaimRejected(profile.GitHubUsername, profile.TrustTier, requestedRole, reason) h.auditRelayAs(profile.GitHubUsername, "", agentaudit.AuditRelayRejected, nil, "reason", "role_claim_rejected") closeWithReason(s.conn, websocket.ClosePolicyViolation, "agent role claim rejected") return true @@ -5196,3 +5197,23 @@ func contributorUsername(c *ContributorConnection) string { } return c.profile.GitHubUsername } + +// roleRejectLogWindow is how long a repeated identical role-claim rejection is +// suppressed from the log. +const roleRejectLogWindow = 10 * time.Minute + +// logRoleClaimRejected logs an expected trust-tier role rejection at Info (it is +// a policy decision, not a hub error) and at most once per (user, role) per +// roleRejectLogWindow. The authorization decision is unchanged. +func (h *ContributeWSHub) logRoleClaimRejected(username, tier, role, reason string) { + key := username + "\x00" + role + now := time.Now() + if last, ok := h.roleRejectLogged.Load(key); ok { + if t, _ := last.(time.Time); now.Sub(t) < roleRejectLogWindow { + return + } + } + h.roleRejectLogged.Store(key, now) + h.logger.Info("[contribute-ws] agent role claim rejected", + "username", username, "tier", tier, "role", role, "reason", reason) +} diff --git a/src/pkg/dashboard/contribute_ws_role_reject_log_test.go b/src/pkg/dashboard/contribute_ws_role_reject_log_test.go new file mode 100644 index 0000000000..facb67851b --- /dev/null +++ b/src/pkg/dashboard/contribute_ws_role_reject_log_test.go @@ -0,0 +1,25 @@ +package dashboard + +import ( + "bytes" + "log/slog" + "strings" + "testing" +) + +// A client retrying a forbidden role must log once per (user, role), at Info. +func TestLogRoleClaimRejectedDedupes(t *testing.T) { + var buf bytes.Buffer + h := &ContributeWSHub{logger: slog.New(slog.NewTextHandler(&buf, nil))} + for i := 0; i < 50; i++ { + h.logRoleClaimRejected("alice", "advisor", "scanner", "nope") + } + h.logRoleClaimRejected("alice", "advisor", "quality", "nope") + out := buf.String() + if n := strings.Count(out, "agent role claim rejected"); n != 2 { + t.Fatalf("got %d log lines, want 2:\n%s", n, out) + } + if strings.Contains(out, "level=WARN") { + t.Fatalf("expected rejection must not log at WARN:\n%s", out) + } +} From 22872c45366e1817f1ef003da1ef4936b5b9b644 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:17:39 -0500 Subject: [PATCH 139/223] =?UTF-8?q?=F0=9F=8C=B1=20test(dashboard):=20recei?= =?UTF-8?q?ver=20accepts=20a=20delivery=20signed=20with=20the=20file-backe?= =?UTF-8?q?d=20webhook=20secret=20(#18082)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #18024 added github.WebhookSecret() (env, else /data/github-webhook-secret.key) and switched the receiver to it, with unit tests in pkg/github. The receiver-level case the issue asked for was missing: with only the secret file present, a correctly signed delivery is accepted (no restart), a wrong signature is rejected, and the env var still wins when both are set. Hive-Run: hivecommons/hive#18025 Hive-Plan: hivecommons/hive#18017 child 1 of 5 Hive-Spec: hivecommons/hive#18025#tests Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .../api_github_webhook_secret_file_test.go | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/pkg/dashboard/api_github_webhook_secret_file_test.go diff --git a/src/pkg/dashboard/api_github_webhook_secret_file_test.go b/src/pkg/dashboard/api_github_webhook_secret_file_test.go new file mode 100644 index 0000000000..ebc7cbb7bd --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_secret_file_test.go @@ -0,0 +1,59 @@ +package dashboard + +import ( + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/github" +) + +// TestGitHubWebhookReceiverAcceptsFileSecret: with no GITHUB_WEBHOOK_SECRET, +// the receiver verifies deliveries against the secret file the owner wrote +// (/data/github-webhook-secret.key in production), with no restart; the env +// var still wins when both are present (hivecommons/hive#18025). +func TestGitHubWebhookReceiverAcceptsFileSecret(t *testing.T) { + const fileSecret = "from-the-file" + t.Setenv(githubWebhookSecretEnvVar, "") + path := filepath.Join(t.TempDir(), "github-webhook-secret.key") + t.Cleanup(github.SetWebhookSecretPathForTest(path)) + t.Cleanup(github.SetWebhookRelayKeyPathForTest(filepath.Join(t.TempDir(), "relay.key"))) + s := newGitHubWebhookTestServer(t) + body := []byte(`{"number":5,"repository":{"full_name":"someone/else"}}`) + + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(fileSecret, body), body); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no env, no file: status = %d, want 503", rec.Code) + } + + if err := os.WriteFile(path, []byte(fileSecret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + // The receiver read (and cached) "no secret" above; a fresh store load + // stands in for the dashboard writing the file through the package. + t.Cleanup(github.SetWebhookSecretPathForTest(path)) + + if got := github.WebhookSecretSource(); got != github.WebhookSecretSourceFile { + t.Fatalf("source = %q, want file", got) + } + if got := github.WebhookVerification(); got != github.WebhookVerificationGitHub { + t.Fatalf("verification = %q, want %q", got, github.WebhookVerificationGitHub) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(fileSecret, body), body); rec.Code != http.StatusOK { + t.Fatalf("file secret, correctly signed: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("wrong", body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("file secret, wrong signature: status = %d, want 401", rec.Code) + } + + t.Setenv(githubWebhookSecretEnvVar, "from-the-env") + if got := github.WebhookSecretSource(); got != github.WebhookSecretSourceEnv { + t.Fatalf("source with env set = %q, want env", got) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(fileSecret, body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("env set, signed with file secret: status = %d, want 401", rec.Code) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("from-the-env", body), body); rec.Code != http.StatusOK { + t.Fatalf("env set, signed with env secret: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } +} From 486085fac6debcaac62a34611fd384e16965d32d Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:31:14 -0500 Subject: [PATCH 140/223] =?UTF-8?q?=F0=9F=8C=B1=20kicks:=20register=20?= =?UTF-8?q?=F0=9F=93=9A=20REFERENCE=20REPOS=20as=20layout=20section=2010a?= =?UTF-8?q?=20and=20golden=20its=20wording=20(#18074)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 kicks: register 📚 REFERENCE REPOS as layout section 10a and golden its wording The reference-repos kick block and its in-scope rule shipped in #18023; this registers it in the kick layout map next to section 10 and pins the wording the scanner's scope classification depends on: - kick_layout.go: document 10a (inside AUTHORIZED REPOS, before the ⛔ scope lines, so "repos not listed above" covers reference repos). - golden for the block with a read-only and an issues+prs entry, plus per-tier labels (read-only, issues, prs, issues+prs). - the cncf/techdocs#395 shape: an item citing only reference repos is listed above the scope line and told it is in scope. - assembled-kick order: reference block after AUTHORIZED REPOS and ahead of the dynamic tail. Closes #18020 Hive-Run: hivecommons/hive#18020 Hive-Plan: hivecommons/hive#18015 child 3 of 5 (kick reference block + in-scope classification) Hive-Spec: reference-repos#kick-wording Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * changelog: fragment for #18074 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/changed-18074.md | 1 + src/pkg/scheduler/agent_repo_scope_test.go | 100 +++++++++++++++++++++ src/pkg/scheduler/kick_layout.go | 3 + src/pkg/scheduler/kick_layout_test.go | 32 +++++++ 4 files changed, 136 insertions(+) create mode 100644 changelog.d/changed-18074.md diff --git a/changelog.d/changed-18074.md b/changelog.d/changed-18074.md new file mode 100644 index 0000000000..beeb0a4829 --- /dev/null +++ b/changelog.d/changed-18074.md @@ -0,0 +1 @@ +- kicks: register REFERENCE REPOS as layout section 10a and golden its wording (#18074) diff --git a/src/pkg/scheduler/agent_repo_scope_test.go b/src/pkg/scheduler/agent_repo_scope_test.go index 40681457a0..4d066fedf2 100644 --- a/src/pkg/scheduler/agent_repo_scope_test.go +++ b/src/pkg/scheduler/agent_repo_scope_test.go @@ -270,3 +270,103 @@ func TestBuildReposSectionFor_ReferenceRepos(t *testing.T) { t.Error("reference block emitted with no reference repos configured") } } + +// referenceReposGolden is the exact 📚 REFERENCE REPOS block for one +// read-only entry and one issues+prs entry (hivecommons/hive#18020). It is a +// golden on purpose: this wording is the only thing that tells the scanner an +// item citing a reference repo is in scope, so a change to it must be a +// deliberate edit here, not a side effect. +const referenceReposGolden = "📚 REFERENCE REPOS (outside this project — study them, do not work them):\n" + + " https://github.com/hivecommons/docs [reference · read-only]\n" + + " https://github.com/hivecommons/hive [reference · issues+prs]\n" + + " You MAY clone them (public, unauthenticated), read their files, and GET their issues/PRs/API. " + + "An authorized-repo item whose deliverables name a reference repo is IN SCOPE, not a scope violation: " + + "do NOT classify it out-of-scope, critical, or a security concern. " + + "Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo (the assessment document, the report), " + + "never as issues or PRs on the reference repo itself — unless its tier above says issues/prs, in which case you may ONLY open an issue, " + + "open a PR from your own branch, or comment on an issue/PR you opened; the proxy refuses merges, default-branch pushes, labels, assignments, " + + "reviews and closing others' items deterministically, so retrying cannot succeed.\n" + +func TestBuildReposSectionFor_ReferenceReposGolden(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + section := s.buildReposSectionFor("scanner") + start := strings.Index(section, "📚 REFERENCE REPOS") + end := strings.Index(section, "⛔ NEVER access") + if start < 0 || end < 0 || end < start { + t.Fatalf("reference block must precede the ⛔ scope line (start=%d end=%d):\n%s", start, end, section) + } + if got := section[start:end]; got != referenceReposGolden { + t.Errorf("reference block drifted from the golden:\n--- got\n%s\n--- want\n%s", got, referenceReposGolden) + } + // It belongs to the scope list, after the authorized repos: "repos not + // listed above" must include the reference repos. + if auth := strings.Index(section, "AUTHORIZED REPOS"); auth < 0 || auth > start { + t.Errorf("reference block must follow AUTHORIZED REPOS:\n%s", section) + } +} + +// Each tier renders on its own: a read-only-only hive never advertises a +// write tier, and an issues-only or prs-only entry prints exactly that tier. +func TestBuildReposSectionFor_ReferenceRepoTiers(t *testing.T) { + for _, tc := range []struct { + write []string + label string + }{ + {nil, "read-only"}, + {[]string{"issues"}, "issues"}, + {[]string{"prs"}, "prs"}, + {[]string{"prs", "issues"}, "issues+prs"}, + } { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{{Repo: "cncf/landscape", Write: tc.write}} + section := s.buildReposSectionFor("scanner") + want := " https://github.com/cncf/landscape [reference · " + tc.label + "]\n" + if !strings.Contains(section, want) { + t.Errorf("write=%v: %q missing:\n%s", tc.write, want, section) + } + if !strings.Contains(section, "IN SCOPE, not a scope violation") { + t.Errorf("write=%v: in-scope rule missing:\n%s", tc.write, section) + } + } +} + +// The cncf/techdocs#395 shape: the hive's only authorized repo is the docs +// repo, and the item's deliverables cite reference repos only. The kick the +// scanner reads lists those repos inside the scope block and tells it the +// item is in scope — the scope classification is the scanner's, driven by +// this text, so this is the classifier contract. +func TestBuildReposSectionFor_ItemCitingOnlyReferenceReposIsInScope(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.Org = "cncf" + s.cfg.Project.Repos = []string{"techdocs"} + s.cfg.Project.PrimaryRepo = "techdocs" + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hivecommons.github.io"}, + {Repo: "hivecommons/hive"}, + } + section := s.buildReposSectionFor("scanner") + never := strings.Index(section, "⛔ NEVER access") + if never < 0 { + t.Fatalf("scope line missing:\n%s", section) + } + for _, cited := range []string{"hivecommons/docs", "hivecommons/hivecommons.github.io", "hivecommons/hive"} { + at := strings.Index(section, "https://github.com/"+cited+" [reference · read-only]") + if at < 0 || at > never { + t.Errorf("%s must be listed above the ⛔ scope line so it is not 'a repo not listed above':\n%s", cited, section) + } + } + for _, want := range []string{ + "https://github.com/cncf/techdocs\n", + "do NOT classify it out-of-scope, critical, or a security concern", + "Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo", + } { + if !strings.Contains(section, want) { + t.Errorf("%q missing:\n%s", want, section) + } + } +} diff --git a/src/pkg/scheduler/kick_layout.go b/src/pkg/scheduler/kick_layout.go index a755a06a7d..8be79d0629 100644 --- a/src/pkg/scheduler/kick_layout.go +++ b/src/pkg/scheduler/kick_layout.go @@ -25,6 +25,9 @@ import ( // 8. <resolved policy body> static text, then ${ISSUE_LIST}/${PR_LIST}/… where the template puts them // 9. REPO-SCOPED CADENCE TARGET semi-static (repo-scoped kicks only) // 10. AUTHORIZED REPOS semi-static (include_repos) +// 10a. 📚 REFERENCE REPOS semi-static (project.reference_repos; rendered inside 10, after +// the authorized/paused lists and before its ⛔ scope lines, so +// "repos not listed above" covers them — #18015/#18020) // 11. hold-gated PR coordination dynamic (held PR snapshot) // 12. red-PR fix-before-new dynamic (ci-failing.json) // 13. review-thread fix-before-new dynamic (review-threads.json) diff --git a/src/pkg/scheduler/kick_layout_test.go b/src/pkg/scheduler/kick_layout_test.go index 971ef98432..db7e0955e1 100644 --- a/src/pkg/scheduler/kick_layout_test.go +++ b/src/pkg/scheduler/kick_layout_test.go @@ -268,3 +268,35 @@ func TestKickLayoutManualKickSharesPrefix(t *testing.T) { t.Errorf("manual kick must not grow the repos block:\n%s", manual) } } + +// Section 10a: project.reference_repos render inside the AUTHORIZED REPOS +// section of the assembled kick, ahead of the dynamic tail +// (hivecommons/hive#18020). +func TestKickLayoutReferenceReposFollowAuthorizedRepos(t *testing.T) { + s := kickLayoutScheduler(t) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + msg := kickLayoutMessage(t, s, makeIssue("test-org/console", 1, "first issue", "scanner", 5, nil, false)) + order := []string{ + "AUTHORIZED REPOS", + "📚 REFERENCE REPOS", + "[reference · read-only]", + "[reference · issues+prs]", + "⛔ NEVER access", + "## Open hold-gated PR coordination", + concreteKickClosingInstruction, + } + last := -1 + for _, want := range order { + at := strings.Index(msg, want) + if at < 0 { + t.Fatalf("kick missing %q:\n%s", want, msg) + } + if at < last { + t.Errorf("%q at %d is out of documented order (previous at %d):\n%s", want, at, last, msg) + } + last = at + } +} From 30c0ca49196339ead0de6351b7e9124405249ef4 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:33:27 -0500 Subject: [PATCH 141/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20gh=202.?= =?UTF-8?q?103.3=20->=202.104.0=20(#18063)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 images: bump gh 2.103.3 -> 2.104.0 Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=2.104.0 SHA256_AMD64=98cb101ad87dbbd3f6b2f2bfcbd43197ccd07643140121d8a9cfaa91eadbc24e SHA256_ARM64=cb34cefc6f4e77b90731374a2b737516bb8f3033c6ca46aa949e5344f68069e3 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * changelog: fragment for #18063 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> --------- Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-18063.md | 1 + src/Dockerfile | 6 +++--- src/deploy/ci-runners/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/changed-18063.md diff --git a/changelog.d/changed-18063.md b/changelog.d/changed-18063.md new file mode 100644 index 0000000000..8aa025ec82 --- /dev/null +++ b/changelog.d/changed-18063.md @@ -0,0 +1 @@ +- images: bump gh 2.103.3 -> 2.104.0 (#18063) diff --git a/src/Dockerfile b/src/Dockerfile index 70476b97bc..274c7caf41 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -245,9 +245,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ chmod +x /usr/local/bin/ttyd # --- Layer 2b: GitHub CLI (off-PATH — only snapshot publisher uses it) --- -ARG GH_VERSION=2.103.3 -ARG GH_SHA256_AMD64=4c806978ffce1cec70853302a56d70ac6da4f19bae7adca4e1a8ab00633dd91d -ARG GH_SHA256_ARM64=ee32df2323c8847f5fac6cb6b4b69ca75828e9329a8f22f677021f7b3306ce63 +ARG GH_VERSION=2.104.0 +ARG GH_SHA256_AMD64=98cb101ad87dbbd3f6b2f2bfcbd43197ccd07643140121d8a9cfaa91eadbc24e +ARG GH_SHA256_ARM64=cb34cefc6f4e77b90731374a2b737516bb8f3033c6ca46aa949e5344f68069e3 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ case "$ARCH" in \ amd64) GH_ARCH=amd64; GH_SHA256="$GH_SHA256_AMD64" ;; \ diff --git a/src/deploy/ci-runners/Dockerfile b/src/deploy/ci-runners/Dockerfile index 73c205c0a8..e9628381ab 100644 --- a/src/deploy/ci-runners/Dockerfile +++ b/src/deploy/ci-runners/Dockerfile @@ -48,7 +48,7 @@ ARG RUNNER_BASE_IMAGE=summerwind/actions-runner:v2.337.0-ubuntu-24.04 FROM ${RUNNER_BASE_IMAGE} -ARG GITHUB_CLI_VERSION=2.103.3 +ARG GITHUB_CLI_VERSION=2.104.0 ARG GITHUB_CLI_KEYRING_SHA256=6084d5d7bd8e288441e0e94fc6275570895da18e6751f70f057485dc2d1a811b # The base image runs as uid 1001 (`runner`), which does not own /etc/apt. From 0124a73288e37bc7882b37d24e997f5ad3fcea38 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 16:41:56 +0000 Subject: [PATCH 142/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.241.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.241.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/changed-18029-webhooks-card-permission-hint.md | 1 - changelog.d/changed-18073.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/changed-18029-webhooks-card-permission-hint.md delete mode 100644 changelog.d/changed-18073.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 03461444e8..5290f196c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.241.1) + +### Changed + +- **Settings → Forge App → GitHub webhooks names the App permission to grant** (#18029): when the GitHub App installation cannot manage hooks, the card now says which permission to grant (Organization → Webhooks `organization_hooks`, or Repository → Webhooks `repository_hooks`, Read and write) so **Create webhook for me** appears, instead of silently omitting the option. Node-executed tests pin the card's copy for relay, none and env-secret modes, and a parity test checks every endpoint the card calls is routed and owner-only. +- docs: project.reference_repos in agent-configuration and hive.yaml.example (#18073) + ## 2026-11-13 (v5.241.0) ### Added diff --git a/changelog.d/changed-18029-webhooks-card-permission-hint.md b/changelog.d/changed-18029-webhooks-card-permission-hint.md deleted file mode 100644 index 8b89d7cf4a..0000000000 --- a/changelog.d/changed-18029-webhooks-card-permission-hint.md +++ /dev/null @@ -1 +0,0 @@ -- **Settings → Forge App → GitHub webhooks names the App permission to grant** (#18029): when the GitHub App installation cannot manage hooks, the card now says which permission to grant (Organization → Webhooks `organization_hooks`, or Repository → Webhooks `repository_hooks`, Read and write) so **Create webhook for me** appears, instead of silently omitting the option. Node-executed tests pin the card's copy for relay, none and env-secret modes, and a parity test checks every endpoint the card calls is routed and owner-only. diff --git a/changelog.d/changed-18073.md b/changelog.d/changed-18073.md deleted file mode 100644 index 82b08a446e..0000000000 --- a/changelog.d/changed-18073.md +++ /dev/null @@ -1 +0,0 @@ -- docs: project.reference_repos in agent-configuration and hive.yaml.example (#18073) From 6a7f22eb0104494631a8675f5f122fe7feb0e5a2 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:53:29 +0000 Subject: [PATCH 143/223] =?UTF-8?q?=E2=9C=A8=20review:=20resolve=20perspec?= =?UTF-8?q?tives=5Fsource=20at=20startup=20and=20reload=20and=20reinstall?= =?UTF-8?q?=20the=20relay=20perspective=20set=20(#18035)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...anged-review-perspectives-source-reload.md | 1 + src/cmd/hive/boot.go | 2 + src/cmd/hive/main.go | 18 +++- src/cmd/hive/main_sweeps.go | 10 ++ .../hive/review_perspectives_source_test.go | 101 ++++++++++++++++++ src/pkg/review/review_heads_decision_test.go | 37 +++++++ 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 changelog.d/changed-review-perspectives-source-reload.md create mode 100644 src/cmd/hive/review_perspectives_source_test.go diff --git a/changelog.d/changed-review-perspectives-source-reload.md b/changelog.d/changed-review-perspectives-source-reload.md new file mode 100644 index 0000000000..c517a8ac1f --- /dev/null +++ b/changelog.d/changed-review-perspectives-source-reload.md @@ -0,0 +1 @@ +- **`review.perspectives_source` is now resolved at startup and on every config reload, and the review relay perspective set is reinstalled each time**, so a verdict naming a perspective the link removed is refused and one it added is accepted without a restart. A dashboard edit of the review config no longer drops the linked set, and a malformed or unreachable document keeps the last-known-good set with a warning. ([#17988](https://github.com/hivecommons/hive/issues/17988), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 91d87cde38..99ed7f91c1 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -31,6 +31,7 @@ import ( "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/perspsrc" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/priorart" "github.com/hivecommons/hive/pkg/promptsrc" @@ -190,6 +191,7 @@ type boot struct { promptFetcher promptsrc.Fetcher defFetcher defsrc.Fetcher definitionResolver *defsrc.Resolver + perspectivesResolver *perspsrc.Resolver pendingTokenSeed []dashboard.TokenSparklineEntry pendingFactSeed []dashboard.FactHistoryEntry pendingCostSeed []dashboard.CostHistoryEntry diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index e58fe9c42a..68a979c6e3 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -61,6 +61,7 @@ import ( "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/maintainernotes" "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/perspsrc" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/policies" "github.com/hivecommons/hive/pkg/promptsrc" @@ -1311,7 +1312,9 @@ func (b *boot) wireBootClosures() { if b.ghClient == nil { return } - installReviewRelaySettings(b.ghClient, b.cfg, b.logger) + // LinkedPerspectives is runtime-only, so re-resolve it before + // the relay reinstall or a dashboard PUT would drop the link. + applyReviewPerspectivesSource(context.Background(), b.cfg, b.perspectivesResolver, b.ghClient, b.logger) // review_bots.min_priority and fix_human_prs are edited // through the same PUT; reinstall so the thread reconciler // applies them without a restart (#10481). @@ -1473,6 +1476,18 @@ func (b *boot) bootGitHubWith(deps bootGitHubDeps) { // The same function every rebuild path uses (#9614). Only the tiers whose // dependencies exist yet apply here; bootAdvisory, bootAgents and // bootDashboard apply the rest as those dependencies come up. + // Resolve review.perspectives_source before the first relay install so + // the relay never boots on a set the link has since replaced. + var persFetcher defsrc.Fetcher + if b.ghClient != nil { + persFetcher = b.ghClient + } + b.perspectivesResolver = perspsrc.NewResolver( + persFetcher, + func(slug string) bool { return b.cfg.GitHubPromptAllowed(slug) }, + b.logger, + ) + perspsrc.ApplyToConfig(b.ctx, b.cfg, b.perspectivesResolver, b.logger) b.configureGitHubClient(b.ghClient) // Long-lived consumers read the client through this provider, never a // captured b.ghClient, so every rebuild reaches them (#9621). @@ -3909,6 +3924,7 @@ func (b *boot) bootWatchersWith(deps bootWatchersDeps) { // fields; a fetch failure keeps each agent's baked definition. Runs before // initAgentConfigDrivenSystems so downstream systems see the merged config. defsrc.ApplyToConfig(context.Background(), b.cfg, b.definitionResolver, b.logger) + applyReviewPerspectivesSource(context.Background(), b.cfg, b.perspectivesResolver, b.ghClient, b.logger) if err := b.cfg.ExpandAgentReplicas(); err != nil { b.logger.Warn("failed to expand agent replicas after config reload", "error", err) } diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 7487b7007d..eebae1bf67 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -26,6 +26,7 @@ import ( "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hooks" "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/perspsrc" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/rotation" @@ -1988,6 +1989,15 @@ func reviewAuthorBase(cfg *config.Config, agent string) string { return agent } +// applyReviewPerspectivesSource re-resolves review.perspectives_source and +// reinstalls the relay settings, so the relay validates verdicts against the +// set the link now names rather than the one it booted with. With no source +// set it fetches nothing and the inline set stands. +func applyReviewPerspectivesSource(ctx context.Context, cfg *config.Config, resolver *perspsrc.Resolver, client *github.Client, logger *slog.Logger) { + perspsrc.ApplyToConfig(ctx, cfg, resolver, logger) + installReviewRelaySettings(client, cfg, logger) +} + // installReviewRelaySettings gives a (possibly rebuilt) GitHub client the // review-relay knobs that live in cfg.Review: which repos may be revised in // place, which perspectives a verdict may name, and whether comments carry a diff --git a/src/cmd/hive/review_perspectives_source_test.go b/src/cmd/hive/review_perspectives_source_test.go new file mode 100644 index 0000000000..d9ae106839 --- /dev/null +++ b/src/cmd/hive/review_perspectives_source_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/perspsrc" + "github.com/hivecommons/hive/pkg/review" +) + +type fakePerspectivesFetcher struct { + doc string + err error + calls int +} + +func (f *fakePerspectivesFetcher) GetFileContentRef(_ context.Context, _, _, _, _ string) (string, error) { + f.calls++ + return f.doc, f.err +} + +const ( + perspectivesDocA = "kind: ReviewPerspectives\nperspectives:\n - name: correctness\n - name: spyre-numerics\n focus: Check numerics on the Spyre path.\n" + perspectivesDocB = "kind: ReviewPerspectives\nperspectives:\n - name: correctness\n" +) + +func perspectivesSourceFixture(f *fakePerspectivesFetcher) (*config.Config, *perspsrc.Resolver, *github.Client, *slog.Logger) { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{} + cfg.Review.PerspectivesSource = &config.DefinitionSourceConfig{Owner: "acme", Repo: "review", Path: "perspectives.yaml"} + resolver := perspsrc.NewResolver(f, func(string) bool { return true }, logger) + client := github.NewClientForTest("http://127.0.0.1:0", "acme", nil, logger) + return cfg, resolver, client, logger +} + +func relayAccepts(cfg *config.Config, logger *slog.Logger, name string) bool { + return reviewPerspectiveSet(cfg, logger).Known(review.Perspective(name)) +} + +func TestApplyReviewPerspectivesSourceReinstallsRelaySet(t *testing.T) { + f := &fakePerspectivesFetcher{doc: perspectivesDocA} + cfg, resolver, client, logger := perspectivesSourceFixture(f) + + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if !relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("relay set should accept spyre-numerics from doc A") + } + + f.doc = perspectivesDocB + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("relay set still accepts spyre-numerics after doc B removed it") + } + if !relayAccepts(cfg, logger, "correctness") { + t.Fatal("relay set lost correctness") + } +} + +func TestApplyReviewPerspectivesSourceMalformedKeepsRelaySet(t *testing.T) { + f := &fakePerspectivesFetcher{doc: perspectivesDocA} + cfg, resolver, client, logger := perspectivesSourceFixture(f) + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + + f.doc = "kind: Nope\n" + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if !relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("malformed document dropped the last-known-good set") + } + if cfg.Review.LinkedPerspectives == nil || cfg.Review.LinkedPerspectives.Warning == "" { + t.Fatalf("want a warning on the kept set, got %+v", cfg.Review.LinkedPerspectives) + } + + f.doc, f.err = "", errors.New("boom") + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if !relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("fetch failure dropped the last-known-good set") + } +} + +func TestApplyReviewPerspectivesSourceUnsetMakesNoFetch(t *testing.T) { + f := &fakePerspectivesFetcher{doc: perspectivesDocA} + cfg, resolver, client, logger := perspectivesSourceFixture(f) + cfg.Review.PerspectivesSource = nil + cfg.Review.Perspectives = []string{"correctness", "security"} + + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if f.calls != 0 { + t.Fatalf("unset perspectives_source fetched %d times", f.calls) + } + if cfg.Review.LinkedPerspectives != nil { + t.Fatalf("LinkedPerspectives = %+v, want nil", cfg.Review.LinkedPerspectives) + } + if !relayAccepts(cfg, logger, "security") || relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("boot relay set changed") + } +} diff --git a/src/pkg/review/review_heads_decision_test.go b/src/pkg/review/review_heads_decision_test.go index 5f5dd86801..46112870dc 100644 --- a/src/pkg/review/review_heads_decision_test.go +++ b/src/pkg/review/review_heads_decision_test.go @@ -362,6 +362,43 @@ func TestRemovedPerspectiveDoesNotBlockHeads(t *testing.T) { } } +// A set that changes through a perspectives_source link is built by +// NewPerspectiveSet from the resolved document; the removed perspective must +// then neither block waiting heads nor count as coverage (#14905, #17988). +func TestLinkedPerspectiveRemovalSettlesWaitingHeads(t *testing.T) { + docB, err := NewPerspectiveSet([]string{"correctness", "security"}, nil) + if err != nil { + t.Fatal(err) + } + opts := AggregateOptions{Perspectives: docB, ExpectedHeads: []string{"cr-sol", "cr-sonnet"}} + linked := Perspective("spyre-numerics") + reports := []PerspectiveReport{ + headRep(PerspectiveCorrectness, "cr-sol", VerdictApprove), + headRep(PerspectiveCorrectness, "cr-sonnet", VerdictApprove), + headRep(PerspectiveSecurity, "cr-sol", VerdictApprove), + headRep(PerspectiveSecurity, "cr-sonnet", VerdictApprove), + headRep(linked, "cr-sol", VerdictChangesRequested, nilDeref), + } + agg := AggregateReports(reports, opts) + if !agg.AwaitingDecision || agg.DecisionSkipped { + t.Fatalf("aggregate = %+v, want awaiting decision", agg) + } + for _, p := range decisionPerspectives(agg, opts.Perspectives) { + if p == linked { + t.Fatalf("decision includes removed linked perspective: %+v", agg) + } + } + + partial := AggregateReports([]PerspectiveReport{ + headRep(PerspectiveCorrectness, "cr-sol", VerdictApprove), + headRep(PerspectiveCorrectness, "cr-sonnet", VerdictApprove), + headRep(linked, "cr-sol", VerdictApprove), + }, opts) + if partial.Verdict != VerdictIncomplete || partial.DecisionSkipped || partial.AwaitingDecision { + t.Fatalf("aggregate = %+v, want incomplete configured coverage", partial) + } +} + func TestRemovedPerspectiveDoesNotCountAsCoverage(t *testing.T) { agg := AggregateReports([]PerspectiveReport{ headRep(PerspectiveCorrectness, "cr-sol", VerdictApprove), From eb8d042f3d6f1b57df216ff112496a6690922929 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:53:43 -0500 Subject: [PATCH 144/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20claude?= =?UTF-8?q?=202.1.322=20->=202.1.323=20(#18065)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=2.1.323 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 274c7caf41..e802b98fed 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -342,7 +342,7 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_V echo "WARN: OpenAI Codex CLI install failed — skipping" # --- Layer 7: Claude Code (most frequent updates) --- -ARG CLAUDE_CODE_VERSION=2.1.322 +ARG CLAUDE_CODE_VERSION=2.1.323 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during the # build) skips claude-code's install.cjs, which is what LINKS the platform # native binary into the package's bin/. The binary itself ships in the diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 0b4f73bf04..bc5e3349ad 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -239,7 +239,7 @@ RUN mkdir -p /etc/apt/keyrings \ # npm cache is cleaned in the same layer it's populated (issue #2613, # Observation 2 "also minor") so the cache doesn't linger in this layer's # diff, mirroring the cleanup already done for the bobshell install below. -ARG CLAUDE_CODE_VERSION=2.1.322 +ARG CLAUDE_CODE_VERSION=2.1.323 ARG COPILOT_VERSION=1.0.88 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during # the build) skips claude-code's install.cjs, which is what LINKS the platform From f84aad60a77a5f3a941ac2275d5b8ded8cd7db0a Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:53:56 -0500 Subject: [PATCH 145/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20reference=5Fti?= =?UTF-8?q?er=20on=20/api/status=20reference=20repos=20(#18021)=20(#18075)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: reference_tier on /api/status reference repos (#18021) project.reference_repos entries in /api/status referenceRepos[] gain an additive reference_tier field (read-only / issues / prs / issues+prs); the existing tier field is kept for older clients and the repos card now prefers reference_tier. A test pins that /api/status reports the tier for both a read-only and an issues+prs entry, that neither leaks into repos[] or the contribute enable/disable list, and that no ACMM, work-item or pause field is present on a reference entry. Docs describe the field; citations re-pointed after the server.go shift. Hive-Run: hivecommons/hive#18021 Hive-Plan: hivecommons/hive#18015 child 4 of 5 (dashboard) Hive-Spec: project.reference_repos#dashboard Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✅ dashboard: make the #18021 reference-leak assertions able to fail The test checked buildRepos and contributeAvailableRepos output with IsReferenceRepo, but newFullServer only manages testorg/testrepo and config validation rejects a repo listed in both project.repos and project.reference_repos, so the checks could never trip. Pin the exact managed set instead; a builder that starts reading reference_repos now fails the test (verified by injecting a reference row). Addresses the P3 review follow-up on hivecommons/hive#18075. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- changelog.d/added-18021-reference-tier.md | 1 + src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 ++--- src/docs/api-reference/snapshots-and-style.md | 2 +- .../reference_repos_status_18021_test.go | 81 +++++++++++++++++++ src/pkg/dashboard/server.go | 4 + .../dashboard/static-src/js/repos-render.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/dashboard/status_builder.go | 3 +- 10 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 changelog.d/added-18021-reference-tier.md create mode 100644 src/pkg/dashboard/reference_repos_status_18021_test.go diff --git a/changelog.d/added-18021-reference-tier.md b/changelog.d/added-18021-reference-tier.md new file mode 100644 index 0000000000..358b39b129 --- /dev/null +++ b/changelog.d/added-18021-reference-tier.md @@ -0,0 +1 @@ +- `/api/status` `referenceRepos[]` entries now carry an additive `reference_tier` field (`read-only`, `issues`, `prs` or `issues+prs`) so API consumers can tell a `project.reference_repos` entry from a managed repo; the entries stay out of `repos[]` and carry no ACMM, work-item or pause fields, and the dashboard's `📚 reference · <tier>` chip reads it ([#18021](https://github.com/hivecommons/hive/issues/18021)). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 07ab917f90..a332a30544 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1704` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1705` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1706` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1708` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1709` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1710` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 0b4e24ec7e..5d10bcbcba 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1711` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1715` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index f26adaa4ae..7d08eb0603 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1684` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1685` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1686` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1692` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1697` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1698` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1699` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1700` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1701` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1702` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1703` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1688` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1689` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1690` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1696` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1701` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1702` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1703` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1704` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1705` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1706` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1707` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 8dde400125..714349077f 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1774` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1778` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/reference_repos_status_18021_test.go b/src/pkg/dashboard/reference_repos_status_18021_test.go new file mode 100644 index 0000000000..ca6dde9572 --- /dev/null +++ b/src/pkg/dashboard/reference_repos_status_18021_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "slices" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +// hivecommons/hive#18021: /api/status lists project.reference_repos apart +// from the managed repos, each with a reference_tier, and with none of the +// ACMM, work-item or pause fields a managed repo row carries. +func TestStatusReferenceReposCarryTierAndNoManagedFields(t *testing.T) { + srv := newFullServer(t) + cfg := srv.deps.Config + cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}}, + } + + // Config validation rejects a repo in both project.repos and + // project.reference_repos, so a reference repo can only leak if the + // builders start reading reference_repos. Pin the exact managed set + // (newFullServer's testorg/testrepo) so that regression fails here. + wantManaged := []string{"testorg/testrepo"} + + repos := buildRepos(cfg, nil, governor.State{}) + var managed []string + for _, r := range repos { + managed = append(managed, r.Full) + } + if !slices.Equal(managed, wantManaged) { + t.Fatalf("managed repos list = %v, want exactly %v (reference repos must not appear)", managed, wantManaged) + } + srv.UpdateStatus(&StatusPayload{Repos: repos, ReferenceRepos: buildReferenceRepos(cfg)}) + + if got := srv.contributeAvailableRepos(); !slices.Equal(got, wantManaged) { + t.Fatalf("contribute enable/disable list = %v, want exactly %v (reference repos must not be offered)", got, wantManaged) + } + + rec := doGet(srv, "/api/status") + if rec.Code != http.StatusOK { + t.Fatalf("/api/status = %d %s", rec.Code, rec.Body.String()) + } + var body struct { + ReferenceRepos []map[string]any `json:"referenceRepos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode /api/status: %v", err) + } + if len(body.ReferenceRepos) != 2 { + t.Fatalf("referenceRepos = %v, want both entries", body.ReferenceRepos) + } + + want := map[string]string{"hivecommons/docs": "read-only", "hivecommons/hive": "issues+prs"} + managedOnly := []string{ + "issues", "prs", "mode", "paused", "acmmLevel", "acmmLevelConfigured", + "acmmPackAgents", "acmmAdvice", "sla", "countsIncomplete", "countsStale", + } + for _, rr := range body.ReferenceRepos { + full, _ := rr["full"].(string) + tier, ok := want[full] + if !ok { + t.Fatalf("unexpected reference entry %v", rr) + } + if got := rr["reference_tier"]; got != tier { + t.Errorf("%s reference_tier = %v, want %q", full, got, tier) + } + if got := rr["tier"]; got != tier { + t.Errorf("%s tier = %v, want %q (kept for existing consumers)", full, got, tier) + } + for _, k := range managedOnly { + if _, present := rr[k]; present { + t.Errorf("%s carries managed-repo field %q: %v", full, k, rr) + } + } + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index a8eb8e7c30..a650064ffc 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -1217,9 +1217,13 @@ type FrontendRestartRefresh struct { // FrontendReferenceRepo is one project.reference_repos entry for the repos // card: the slug and its tier ("read-only", "issues", "prs", "issues+prs"). +// It deliberately carries no ACMM, work-item or pause fields (#18021). type FrontendReferenceRepo struct { Full string `json:"full"` Tier string `json:"tier"` + // ReferenceTier repeats Tier under the name API consumers key on to tell + // a reference entry from a managed repo row (hivecommons/hive#18021). + ReferenceTier string `json:"reference_tier"` } type FrontendRepo struct { diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index c8f98b6833..b6f16b1201 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -365,7 +365,7 @@ const base = window._githubBaseUrl || 'https://github.com'; return refs.map(rr => { const full = String(rr.full || ''); - const tier = String(rr.tier || 'read-only'); + const tier = String(rr.reference_tier || rr.tier || 'read-only'); const tip = tier === 'read-only' ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index a59be35a8e..2bc465b916 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -37143,7 +37143,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const base = window._githubBaseUrl || 'https://github.com'; return refs.map(rr => { const full = String(rr.full || ''); - const tier = String(rr.tier || 'read-only'); + const tier = String(rr.reference_tier || rr.tier || 'read-only'); const tip = tier === 'read-only' ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 40e031e0b0..18be7c3f88 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -2381,7 +2381,8 @@ func buildReferenceRepos(cfg *config.Config) []FrontendReferenceRepo { } out := make([]FrontendReferenceRepo, 0, len(cfg.Project.ReferenceRepos)) for _, rr := range cfg.Project.ReferenceRepos { - out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: rr.WriteLabel()}) + tier := rr.WriteLabel() + out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: tier, ReferenceTier: tier}) } return out } From ca21c90d74b35ea98f659fc0fc5aa9aa88427612 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:54:30 -0500 Subject: [PATCH 146/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20idempotent=20w?= =?UTF-8?q?ebhook=20create=20through=20the=20App,=20412=20on=20missing=20h?= =?UTF-8?q?ook=20permission=20(#18086)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/config/github/webhooks/create now lists the target's hooks first and returns an existing hook with the same config.url (created: false) instead of creating a duplicate; a 422 "already exists" race resolves to the winner the same way. A missing organization_hooks / repository_hooks write permission answers 412 naming the permission. The response carries id, events, payload_url and created next to the existing fields; only an actual creation is audited. The Settings card says "Already set up" when the hook was found. Tests use an httptest GitHub stub: 409 without a secret, 412 without the permission, create on first call and return existing on second, repo scope hits the repo endpoint. Hive-Run: hivecommons/hive#18028 Hive-Plan: hivecommons/hive#18017 child 3 of 5 (webhook create endpoint) Hive-Spec: hivecommons/hive#18028#webhooks-create Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...changed-18028-webhook-create-idempotent.md | 1 + dashboard/openapi.json | 30 ++- src/docs/api-reference/configuration.md | 6 +- src/docs/webhooks.md | 4 +- src/pkg/dashboard/api_github_webhook_setup.go | 19 +- .../api_github_webhook_setup_test.go | 181 +++++++++++++++++- .../static-src/js/config-webhooks.js | 5 +- src/pkg/dashboard/static/index.html | 5 +- src/pkg/github/webhook_setup.go | 107 +++++++++-- src/pkg/github/webhook_setup_test.go | 153 +++++++++++++++ 10 files changed, 478 insertions(+), 33 deletions(-) create mode 100644 changelog.d/changed-18028-webhook-create-idempotent.md create mode 100644 src/pkg/github/webhook_setup_test.go diff --git a/changelog.d/changed-18028-webhook-create-idempotent.md b/changelog.d/changed-18028-webhook-create-idempotent.md new file mode 100644 index 0000000000..01d2c011f1 --- /dev/null +++ b/changelog.d/changed-18028-webhook-create-idempotent.md @@ -0,0 +1 @@ +- Dashboard: "Create webhook for me" is idempotent — it returns the existing hook for this hive's payload URL instead of creating a duplicate, and answers 412 naming the missing `organization_hooks`/`repository_hooks` write permission (#18028) diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 6167fff8a0..9d1e34573a 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -25719,7 +25719,7 @@ "System" ], "summary": "Create the org or repo webhook through the GitHub App (owner)", - "description": "Owner-only. Uses the App installation's organization_hooks or repository_hooks write permission to create a webhook on GitHub pointing at this hive's payload URL with the configured secret and event list. Requires a configured secret, a public dashboard URL and the matching permission; the repository must be one of project.repos. Audited.", + "description": "Owner-only. Uses the App installation's organization_hooks or repository_hooks write permission to create a webhook on GitHub pointing at this hive's payload URL with the configured secret and event list. Requires a configured secret, a public dashboard URL and the matching permission; the repository must be one of project.repos. Idempotent: when a hook with the same payload URL already exists on the target it is returned with created=false instead of creating a duplicate. Creation is audited.", "requestBody": { "required": true, "content": { @@ -25745,15 +25745,32 @@ }, "responses": { "200": { - "description": "Webhook created", + "description": "Webhook created, or the existing hook for this payload URL", "content": { "application/json": { "schema": { "type": "object", "properties": { - "hook_id": { + "id": { "type": "integer" }, + "hook_id": { + "type": "integer", + "description": "Same as id" + }, + "created": { + "type": "boolean", + "description": "false when an existing hook for this payload URL was returned" + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "payload_url": { + "type": "string" + }, "scope": { "type": "string" }, @@ -25772,10 +25789,13 @@ "description": "Invalid scope or repository" }, "403": { - "description": "Not an owner, or the App lacks the hooks permission" + "description": "Not an owner" }, "409": { - "description": "No secret / no public URL / webhook already exists" + "description": "No secret / no public URL / no project.org / no App auth" + }, + "412": { + "description": "The App installation lacks organization_hooks or repository_hooks write for the scope" }, "502": { "description": "GitHub API error" diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 02117e6006..c11dea097d 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -75,9 +75,9 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:314` | | `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:315` | | `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:318` | -| `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:87` | -| `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:88` | -| `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:89` | +| `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:88` | +| `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:89` | +| `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:90` | | `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:376` | | `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:377` | | `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:378` | diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 37865c7593..ab1d9ade5e 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -53,7 +53,9 @@ uses a secret only you hold. 2. **Create the webhook.** If the section offers **Create webhook for me**, click it: the hive's GitHub App installation holds `organization_hooks` or `repository_hooks` write permission and the hive creates the hook with the - right URL, secret and events. Otherwise do it by hand in GitHub: + right URL, secret and events. Clicking it again is safe: when a hook for + this hive's payload URL already exists it is reported, not duplicated. + Otherwise do it by hand in GitHub: organization (or repository) → Settings → Webhooks → **Add webhook**. - Payload URL: the value shown in the section, i.e. `https://<your-hive>/api/webhook/github` diff --git a/src/pkg/dashboard/api_github_webhook_setup.go b/src/pkg/dashboard/api_github_webhook_setup.go index 640157b176..4c38e68487 100644 --- a/src/pkg/dashboard/api_github_webhook_setup.go +++ b/src/pkg/dashboard/api_github_webhook_setup.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net/http" + "strconv" "strings" "time" @@ -246,6 +247,9 @@ type webhookCreateBody struct { // handleConfigGitHubWebhooksCreate serves POST /api/config/github/webhooks/create // (owner-only): creates the org or repo hook through the App installation. +// 409 without a webhook secret, 412 when the installation lacks the hooks +// write permission for the scope; when a hook for this hive's payload URL +// already exists it is returned with created=false instead of duplicated. func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http.Request) { if !requireOwnerRole(w, r) { return @@ -293,12 +297,12 @@ func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http switch strings.ToLower(strings.TrimSpace(body.Scope)) { case "", "organization", "org": if !orgHooks { - jsonError(w, "the App installation has no organization_hooks: write permission", http.StatusForbidden) + jsonError(w, "the App installation lacks organization_hooks: write permission — grant it on the App and accept the installation update", http.StatusPreconditionFailed) return } case "repository", "repo": if !repoHooks { - jsonError(w, "the App installation has no repository_hooks: write permission", http.StatusForbidden) + jsonError(w, "the App installation lacks repository_hooks: write permission — grant it on the App and accept the installation update", http.StatusPreconditionFailed) return } repo := strings.TrimSpace(body.Repo) @@ -324,8 +328,15 @@ func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http return } user := strings.TrimSpace(r.Header.Get("X-Hive-User")) - s.audit.LogRecord(user, auditWebhookCreated, auditDetail("scope", res.Scope, "target", res.Target, "payload_url", payloadURL), "", res.Target, 0) - s.webhookLogger().Info("github webhook created through the App", "user", user, "scope", res.Scope, "target", res.Target) + if !res.Created { + // Idempotent: a hook for this payload URL already exists on the + // target; nothing was mutated, so there is nothing to audit. + s.webhookLogger().Info("github webhook already present; returning it", "user", user, "scope", res.Scope, "target", res.Target, "hook_id", res.ID) + jsonResponse(w, res) + return + } + s.audit.LogRecord(user, auditWebhookCreated, auditDetail("scope", res.Scope, "target", res.Target, "payload_url", payloadURL, "hook_id", strconv.FormatInt(res.ID, 10)), "", res.Target, 0) + s.webhookLogger().Info("github webhook created through the App", "user", user, "scope", res.Scope, "target", res.Target, "hook_id", res.ID) jsonResponse(w, res) } diff --git a/src/pkg/dashboard/api_github_webhook_setup_test.go b/src/pkg/dashboard/api_github_webhook_setup_test.go index f657d89057..856c2b7346 100644 --- a/src/pkg/dashboard/api_github_webhook_setup_test.go +++ b/src/pkg/dashboard/api_github_webhook_setup_test.go @@ -2,12 +2,22 @@ package dashboard import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" "encoding/json" + "encoding/pem" + "fmt" + "io" + "log/slog" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" + "sync" "testing" + "time" ghpkg "github.com/hivecommons/hive/pkg/github" ) @@ -166,7 +176,7 @@ func TestWebhookSetup_CreateGatedOnAppPermission(t *testing.T) { srv.mux.ServeHTTP(rec, req) return rec } - if rec := post(`{"scope":"organization"}`); rec.Code != http.StatusForbidden { + if rec := post(`{"scope":"organization"}`); rec.Code != http.StatusPreconditionFailed { t.Fatalf("org scope without organization_hooks: %d %s", rec.Code, rec.Body.String()) } if rec := post(`{"scope":"repository","repo":"not-ours"}`); rec.Code != http.StatusBadRequest { @@ -181,3 +191,172 @@ func TestWebhookSetup_CreateGatedOnAppPermission(t *testing.T) { t.Fatal("create offered without any hooks permission") } } + +// webhookGitHubStub serves the installation-token mint plus the org/repo +// hooks list+create endpoints, remembering created hooks per path. +type webhookGitHubStub struct { + mu sync.Mutex + hooks map[string][]map[string]any + posts map[string]int +} + +func (g *webhookGitHubStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + g.mu.Lock() + defer g.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + path := strings.TrimSuffix(r.URL.Path, "/") + switch { + case strings.HasSuffix(path, "/access_tokens"): + w.WriteHeader(http.StatusCreated) + fmt.Fprintf(w, `{"token":"ghs_webhook_stub","expires_at":%q}`, time.Now().Add(time.Hour).UTC().Format(time.RFC3339)) + case strings.HasSuffix(path, "/hooks") && r.Method == http.MethodGet: + list := g.hooks[path] + if list == nil { + list = []map[string]any{} + } + _ = json.NewEncoder(w).Encode(list) + case strings.HasSuffix(path, "/hooks") && r.Method == http.MethodPost: + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + g.posts[path]++ + hook := map[string]any{"id": 4200 + len(g.hooks[path]) + g.posts[path], "events": body["events"], "config": body["config"], "active": true} + g.hooks[path] = append(g.hooks[path], hook) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(hook) + default: + http.NotFound(w, r) + } +} + +func (g *webhookGitHubStub) postCount(path string) int { + g.mu.Lock() + defer g.mu.Unlock() + return g.posts[path] +} + +func (g *webhookGitHubStub) totalPosts() int { + g.mu.Lock() + defer g.mu.Unlock() + n := 0 + for _, c := range g.posts { + n += c + } + return n +} + +func newWebhookCreateServer(t *testing.T, perms map[string]string) (*Server, *webhookGitHubStub) { + t.Helper() + stub := &webhookGitHubStub{hooks: map[string][]map[string]any{}, posts: map[string]int{}} + api := httptest.NewServer(stub) + t.Cleanup(api.Close) + + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate key: %v", err) + } + keyFile := filepath.Join(t.TempDir(), "app-key.pem") + if err := os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + auth, err := ghpkg.NewAppAuthWithCache(1, 2, keyFile, filepath.Join(t.TempDir(), "token.cache"), + slog.New(slog.NewTextHandler(io.Discard, nil)), api.URL) + if err != nil { + t.Fatalf("NewAppAuthWithCache: %v", err) + } + + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "https://hive.example.org" + srv.deps.Config.Project.Org = "projectbluefin" + srv.deps.Config.Project.Repos = []string{"bluefin", "projectbluefin/common"} + srv.deps.GHAppAuth = auth + orig := webhookPermissionProbe + t.Cleanup(func() { webhookPermissionProbe = orig }) + webhookPermissionProbe = func(context.Context, *ghpkg.AppAuth) (*ghpkg.InstallationInfo, error) { + return &ghpkg.InstallationInfo{Permissions: perms}, nil + } + return srv, stub +} + +func postWebhookCreate(srv *Server, payload string) (*httptest.ResponseRecorder, ghpkg.WebhookCreateResult) { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/create", strings.NewReader(payload)) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + var res ghpkg.WebhookCreateResult + if rec.Code == http.StatusOK { + _ = json.Unmarshal(rec.Body.Bytes(), &res) + } + return rec, res +} + +func TestWebhookCreate_NoSecretIs409(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "") + restore := ghpkg.SetWebhookSecretPathForTest(filepath.Join(t.TempDir(), "absent.key")) + defer restore() + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionOrganizationHooks: "write"}) + if rec, _ := postWebhookCreate(srv, `{"scope":"org"}`); rec.Code != http.StatusConflict { + t.Fatalf("no secret: %d %s, want 409", rec.Code, rec.Body.String()) + } + if stub.totalPosts() != 0 { + t.Fatal("GitHub hook created without a secret") + } +} + +func TestWebhookCreate_MissingPermissionIs412(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionOrganizationHooks: "read"}) + rec, _ := postWebhookCreate(srv, `{"scope":"org"}`) + if rec.Code != http.StatusPreconditionFailed || !strings.Contains(rec.Body.String(), "organization_hooks") { + t.Fatalf("org without write: %d %s, want 412 naming organization_hooks", rec.Code, rec.Body.String()) + } + rec, _ = postWebhookCreate(srv, `{"scope":"repo","repo":"projectbluefin/bluefin"}`) + if rec.Code != http.StatusPreconditionFailed || !strings.Contains(rec.Body.String(), "repository_hooks") { + t.Fatalf("repo without write: %d %s, want 412 naming repository_hooks", rec.Code, rec.Body.String()) + } + if stub.totalPosts() != 0 { + t.Fatal("GitHub hook created without the permission") + } +} + +// First call creates the org hook (audited); the second returns it instead +// of creating a duplicate. +func TestWebhookCreate_OrgCreatesThenReturnsExisting(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionOrganizationHooks: "write"}) + + rec, first := postWebhookCreate(srv, `{"scope":"org"}`) + if rec.Code != http.StatusOK || !first.Created || first.ID == 0 || len(first.Events) != len(ghpkg.WebhookEvents) { + t.Fatalf("first create: %d %s", rec.Code, rec.Body.String()) + } + if first.PayloadURL != "https://hive.example.org"+githubWebhookPath { + t.Fatalf("payload url = %q", first.PayloadURL) + } + rec, second := postWebhookCreate(srv, `{"scope":"org"}`) + if rec.Code != http.StatusOK || second.Created || second.ID != first.ID { + t.Fatalf("second create: %d %s, want existing hook %d", rec.Code, rec.Body.String(), first.ID) + } + if n := stub.postCount("/orgs/projectbluefin/hooks"); n != 1 { + t.Fatalf("org POSTs = %d, want 1", n) + } + var audited int + for _, e := range srv.audit.Recent(50) { + if e.Action == auditWebhookCreated { + audited++ + } + } + if audited != 1 { + t.Fatalf("audit entries = %d, want 1 (only the real creation)", audited) + } +} + +func TestWebhookCreate_RepoScopeHitsRepoEndpoint(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionRepositoryHooks: "write"}) + rec, res := postWebhookCreate(srv, `{"scope":"repo","repo":"projectbluefin/common"}`) + if rec.Code != http.StatusOK || !res.Created || res.Target != "projectbluefin/common" { + t.Fatalf("repo create: %d %s", rec.Code, rec.Body.String()) + } + if stub.postCount("/repos/projectbluefin/common/hooks") != 1 || stub.postCount("/orgs/projectbluefin/hooks") != 0 { + t.Fatal("want exactly one POST, to the repo endpoint") + } +} diff --git a/src/pkg/dashboard/static-src/js/config-webhooks.js b/src/pkg/dashboard/static-src/js/config-webhooks.js index 257a1ba44a..18cc6682ec 100644 --- a/src/pkg/dashboard/static-src/js/config-webhooks.js +++ b/src/pkg/dashboard/static-src/js/config-webhooks.js @@ -202,8 +202,9 @@ }); const body = await res.json().catch(() => ({})); if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); - if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; - showToast('Webhook created', 'info'); + const verb = body.created === false ? 'Already set up:' : 'Created'; + if (out) out.innerHTML = `<span class="wh-ok">${verb} ${esc(body.scope)} hook #${esc(String(body.id || body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast(body.created === false ? 'Webhook already exists' : 'Webhook created', 'info'); } catch (e) { if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; } finally { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 2bc465b916..01999d74af 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -59881,8 +59881,9 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> }); const body = await res.json().catch(() => ({})); if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); - if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; - showToast('Webhook created', 'info'); + const verb = body.created === false ? 'Already set up:' : 'Created'; + if (out) out.innerHTML = `<span class="wh-ok">${verb} ${esc(body.scope)} hook #${esc(String(body.id || body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast(body.created === false ? 'Webhook already exists' : 'Webhook created', 'info'); } catch (e) { if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; } finally { diff --git a/src/pkg/github/webhook_setup.go b/src/pkg/github/webhook_setup.go index 78aaeda11f..0e95da85a6 100644 --- a/src/pkg/github/webhook_setup.go +++ b/src/pkg/github/webhook_setup.go @@ -68,18 +68,32 @@ type WebhookCreateRequest struct { Events []string } -// WebhookCreateResult names the hook GitHub created. +// WebhookCreateResult names the hook CreateWebhook created or found. type WebhookCreateResult struct { - HookID int64 `json:"hook_id"` - Scope string `json:"scope"` // "organization" or "repository" - Target string `json:"target"` - URL string `json:"url"` // GitHub settings page for the hook + // ID is the GitHub hook id; HookID repeats it for the Settings card. + ID int64 `json:"id"` + HookID int64 `json:"hook_id"` + Scope string `json:"scope"` // "organization" or "repository" + Target string `json:"target"` + URL string `json:"url"` // GitHub settings page for the hook + Events []string `json:"events"` + // PayloadURL is the hook's config.url (this hive's receiver). + PayloadURL string `json:"payload_url"` + // Created is false when a hook with the same payload URL already existed + // on the target and was returned instead of creating a duplicate. + Created bool `json:"created"` } +// webhookListPages bounds the hook listing; GitHub caps hooks per target at +// 20 for repositories and organizations, so one page is normally enough. +const webhookListPages = 5 + // CreateWebhook creates an organization hook (req.Repo == "") or a repository -// hook through the App installation token. The token carries every granted -// permission, so the caller checks WebhookHookPermissions first and only -// offers the action when the matching write permission is held. +// hook through the App installation token. It is idempotent: the target's +// hooks are listed first and a hook whose config.url equals req.PayloadURL is +// returned (Created=false) instead of creating a duplicate. The token carries +// every granted permission, so the caller checks WebhookHookPermissions first +// and only offers the action when the matching write permission is held. func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) (*WebhookCreateResult, error) { if a == nil { return nil, errors.New("GitHub App auth not configured") @@ -96,6 +110,15 @@ func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) ( if len(events) == 0 { events = WebhookEvents } + + existing, err := findWebhookByURL(ctx, client, req.Org, req.Repo, req.PayloadURL) + if err != nil { + return nil, fmt.Errorf("listing existing hooks: %w", err) + } + if existing != nil { + return a.webhookResult(req, existing, false), nil + } + hook := &gh.Hook{ Active: gh.Ptr(true), Events: events, @@ -110,28 +133,82 @@ func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) ( created *gh.Hook resp *gh.Response ) - res := &WebhookCreateResult{} if req.Repo == "" { created, resp, err = client.Organizations.CreateHook(ctx, req.Org, hook) - res.Scope, res.Target = "organization", req.Org } else { created, resp, err = client.Repositories.CreateHook(ctx, req.Org, req.Repo, hook) - res.Scope, res.Target = "repository", req.Org+"/"+req.Repo } if err != nil { if resp != nil && resp.StatusCode == http.StatusUnprocessableEntity && strings.Contains(strings.ToLower(err.Error()), "already exists") { + // Lost a race with a concurrent create: return the winner. + if found, ferr := findWebhookByURL(ctx, client, req.Org, req.Repo, req.PayloadURL); ferr == nil && found != nil { + return a.webhookResult(req, found, false), nil + } return nil, ErrWebhookExists } return nil, err } - res.HookID = created.GetID() + if len(created.Events) == 0 { + created.Events = events + } + return a.webhookResult(req, created, true), nil +} + +// findWebhookByURL returns the hook on the org (repo == "") or repository +// whose config.url equals payloadURL, or nil when there is none. +func findWebhookByURL(ctx context.Context, client *gh.Client, org, repo, payloadURL string) (*gh.Hook, error) { + want := strings.TrimRight(strings.TrimSpace(payloadURL), "/") + opts := &gh.ListOptions{PerPage: 100} + for page := 0; page < webhookListPages; page++ { + var ( + hooks []*gh.Hook + resp *gh.Response + err error + ) + if repo == "" { + hooks, resp, err = client.Organizations.ListHooks(ctx, org, opts) + } else { + hooks, resp, err = client.Repositories.ListHooks(ctx, org, repo, opts) + } + if err != nil { + return nil, err + } + for _, h := range hooks { + if h.GetConfig() == nil { + continue + } + if strings.TrimRight(strings.TrimSpace(h.GetConfig().GetURL()), "/") == want { + return h, nil + } + } + if resp == nil || resp.NextPage == 0 { + return nil, nil + } + opts.Page = resp.NextPage + } + return nil, nil +} + +func (a *AppAuth) webhookResult(req WebhookCreateRequest, hook *gh.Hook, created bool) *WebhookCreateResult { + res := &WebhookCreateResult{ + ID: hook.GetID(), + HookID: hook.GetID(), + Events: append([]string(nil), hook.Events...), + PayloadURL: req.PayloadURL, + Created: created, + } + if hook.GetConfig() != nil && hook.GetConfig().GetURL() != "" { + res.PayloadURL = hook.GetConfig().GetURL() + } base := strings.TrimSuffix(webBaseURLForAPI(a.apiURL), "/") if req.Repo == "" { - res.URL = fmt.Sprintf("%s/organizations/%s/settings/hooks/%d", base, req.Org, res.HookID) + res.Scope, res.Target = "organization", req.Org + res.URL = fmt.Sprintf("%s/organizations/%s/settings/hooks/%d", base, req.Org, res.ID) } else { - res.URL = fmt.Sprintf("%s/%s/%s/settings/hooks/%d", base, req.Org, req.Repo, res.HookID) + res.Scope, res.Target = "repository", req.Org+"/"+req.Repo + res.URL = fmt.Sprintf("%s/%s/%s/settings/hooks/%d", base, req.Org, req.Repo, res.ID) } - return res, nil + return res } // webBaseURLForAPI maps an API URL to the web origin a hook's settings page diff --git a/src/pkg/github/webhook_setup_test.go b/src/pkg/github/webhook_setup_test.go new file mode 100644 index 0000000000..2a4f86b737 --- /dev/null +++ b/src/pkg/github/webhook_setup_test.go @@ -0,0 +1,153 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// hookStub is a minimal GitHub hooks API: GET/POST on /orgs/{org}/hooks and +// /repos/{owner}/{repo}/hooks, keeping created hooks per path. +type hookStub struct { + mu sync.Mutex + hooks map[string][]map[string]any + posts map[string]int + nextID int64 + lastReq map[string]any +} + +func newHookStub() *hookStub { + return &hookStub{hooks: map[string][]map[string]any{}, posts: map[string]int{}, nextID: 100} +} + +func (s *hookStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + path := strings.TrimSuffix(r.URL.Path, "/") + if !strings.HasSuffix(path, "/hooks") { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + switch r.Method { + case http.MethodGet: + list := s.hooks[path] + if list == nil { + list = []map[string]any{} + } + _ = json.NewEncoder(w).Encode(list) + case http.MethodPost: + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + s.lastReq = body + s.posts[path]++ + s.nextID++ + hook := map[string]any{ + "id": s.nextID, + "active": body["active"], + "events": body["events"], + "config": body["config"], + } + s.hooks[path] = append(s.hooks[path], hook) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(hook) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func (s *hookStub) postCount(path string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.posts[path] +} + +func (s *hookStub) lastBody() map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + return s.lastReq +} + +func hookStubAuth(t *testing.T, stub http.Handler) *AppAuth { + t.Helper() + srv := httptest.NewServer(stub) + t.Cleanup(srv.Close) + return &AppAuth{ + cachedToken: "ghs_hook_stub", + tokenExpiry: time.Now().Add(2 * time.Hour), + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + installationID: 1, + apiURL: srv.URL + "/", + } +} + +func TestCreateWebhook_OrgCreatesThenReturnsExisting(t *testing.T) { + stub := newHookStub() + auth := hookStubAuth(t, stub) + req := WebhookCreateRequest{Org: "acme", PayloadURL: "https://hive.example.org/api/webhook/github", Secret: "s3cret"} + + res, err := auth.CreateWebhook(context.Background(), req) + if err != nil { + t.Fatalf("first create: %v", err) + } + if !res.Created || res.ID == 0 || res.ID != res.HookID || res.Scope != "organization" || res.Target != "acme" { + t.Fatalf("first result = %+v", res) + } + if len(res.Events) != len(WebhookEvents) { + t.Fatalf("events = %v, want default WebhookEvents", res.Events) + } + body := stub.lastBody() + cfg, _ := body["config"].(map[string]any) + if cfg["url"] != req.PayloadURL || cfg["content_type"] != "json" || cfg["secret"] != "s3cret" || cfg["insecure_ssl"] != "0" || body["active"] != true { + t.Fatalf("create body = %v", body) + } + + again, err := auth.CreateWebhook(context.Background(), req) + if err != nil { + t.Fatalf("second create: %v", err) + } + if again.Created || again.ID != res.ID { + t.Fatalf("second result = %+v, want existing hook %d", again, res.ID) + } + if n := stub.postCount("/orgs/acme/hooks"); n != 1 { + t.Fatalf("POST count = %d, want 1 (no duplicate)", n) + } +} + +func TestCreateWebhook_RepoScopeHitsRepoEndpoint(t *testing.T) { + stub := newHookStub() + auth := hookStubAuth(t, stub) + res, err := auth.CreateWebhook(context.Background(), WebhookCreateRequest{ + Org: "acme", Repo: "widgets", PayloadURL: "https://hive.example.org/api/webhook/github", Secret: "s3cret", + }) + if err != nil { + t.Fatalf("create: %v", err) + } + if stub.postCount("/repos/acme/widgets/hooks") != 1 || stub.postCount("/orgs/acme/hooks") != 0 { + t.Fatal("want exactly one POST, to the repo endpoint") + } + if !res.Created || res.Scope != "repository" || res.Target != "acme/widgets" || !strings.HasSuffix(res.URL, "/acme/widgets/settings/hooks/101") { + t.Fatalf("result = %+v", res) + } +} + +// A hook for another URL on the same target does not count as existing. +func TestCreateWebhook_OtherURLDoesNotMatch(t *testing.T) { + stub := newHookStub() + stub.hooks["/orgs/acme/hooks"] = []map[string]any{{"id": 7, "config": map[string]any{"url": "https://ci.example.org/hook"}}} + auth := hookStubAuth(t, stub) + res, err := auth.CreateWebhook(context.Background(), WebhookCreateRequest{Org: "acme", PayloadURL: "https://hive.example.org/api/webhook/github", Secret: "x"}) + if err != nil { + t.Fatalf("create: %v", err) + } + if !res.Created || res.ID == 7 { + t.Fatalf("result = %+v, want a new hook", res) + } +} From a96dc544b20cf7f578049b80730631666429e59c Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:54:51 -0500 Subject: [PATCH 147/223] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20stop=20a=20?= =?UTF-8?q?Hold=20click=20down=20the=20Projects=20card=20from=20releasing?= =?UTF-8?q?=20a=20neighbour=20(#18085)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hold toggle moves its row into the held group and the status render that follows re-sorts the groups, so the button under the pointer changes a moment after every click. Working down a card clicking Hold, the next click landed on another item's Release and that item's hold came straight off: on cncf/techdocs #151 was released in the same second #283 was held, #153 three seconds after #54, #221 two seconds after #139 — every label removal was the hive bot acting on a dashboard Release. renderRepos now notes when the card's rows move (order or membership), and a Release that arrives within 1.5s of a move is not sent; a toast tells the operator to click Release again. Hold is never delayed. The in-flight guard moves from the clicked button (which the optimistic repaint replaces) to the item, so one item never has two toggles racing. Fixes #18054 Hive-Run: hivecommons/hive#18054 Hive-Plan: issue-18054 Hive-Spec: issue-18054#hold-sticks Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: nate-double-u <4453979+nate-double-u@users.noreply.github.com> --- changelog.d/fixed-18054-hold-release-guard.md | 1 + .../repo_card_held_pills_7896_test.go | 2 + .../repo_hold_shift_guard_18054_test.go | 164 ++++++++++++++++++ .../dashboard/static-src/js/repos-render.js | 1 + src/pkg/dashboard/static-src/js/repos.js | 40 +++++ src/pkg/dashboard/static/index.html | 41 +++++ 6 files changed, 249 insertions(+) create mode 100644 changelog.d/fixed-18054-hold-release-guard.md create mode 100644 src/pkg/dashboard/repo_hold_shift_guard_18054_test.go diff --git a/changelog.d/fixed-18054-hold-release-guard.md b/changelog.d/fixed-18054-hold-release-guard.md new file mode 100644 index 0000000000..6e43c24352 --- /dev/null +++ b/changelog.d/fixed-18054-hold-release-guard.md @@ -0,0 +1 @@ +- dashboard: a Projects-card Release that lands while the rows are re-sorting after a Hold is no longer sent, so holding items down a list no longer releases a neighbouring item's hold (#18054) diff --git a/src/pkg/dashboard/repo_card_held_pills_7896_test.go b/src/pkg/dashboard/repo_card_held_pills_7896_test.go index 9d7899debc..b51013ca28 100644 --- a/src/pkg/dashboard/repo_card_held_pills_7896_test.go +++ b/src/pkg/dashboard/repo_card_held_pills_7896_test.go @@ -271,6 +271,7 @@ func TestRepoHoldToggleOptimisticRollback(t *testing.T) { jsFunc(t, html, "repoParts") + "\n" + jsFunc(t, html, "updateRepoBreakdownForHold") + "\n" + jsFunc(t, html, "moveRepoItemHold") + "\n" + + repoHoldShiftGuardJS(t, html) + "\n" + jsFunc(t, html, "toggleRepoItemHold") + "\n" + holdToggleRollbackAssertions path := filepath.Join(t.TempDir(), "hold-toggle.js") @@ -305,6 +306,7 @@ func TestRepoHoldToggleSuccessRaisesStatusFloor(t *testing.T) { jsFunc(t, html, "repoParts") + "\n" + jsFunc(t, html, "updateRepoBreakdownForHold") + "\n" + jsFunc(t, html, "moveRepoItemHold") + "\n" + + repoHoldShiftGuardJS(t, html) + "\n" + jsFunc(t, html, "toggleRepoItemHold") + "\n" + holdToggleSuccessAssertions path := filepath.Join(t.TempDir(), "hold-toggle-success.js") diff --git a/src/pkg/dashboard/repo_hold_shift_guard_18054_test.go b/src/pkg/dashboard/repo_hold_shift_guard_18054_test.go new file mode 100644 index 0000000000..665ecb135d --- /dev/null +++ b/src/pkg/dashboard/repo_hold_shift_guard_18054_test.go @@ -0,0 +1,164 @@ +package dashboard + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// hivecommons/hive#18054: on the Projects card, Hold on cncf/techdocs #151, +// #153 and #221 "did not stick". GitHub's timeline showed why: each hold +// label came off again a moment after a Hold click on a DIFFERENT row (#151 +// released in the same second #283 was held, #153 three seconds after #54, +// #221 two seconds after #139). A hold toggle moves its row into the held +// group and the next status render re-sorts the groups, so the operator +// working down the list hit another item's Release button. These tests pin +// the guard: a Release that lands while the rows are moving is not sent, a +// Hold still is, and one item never has two toggles in flight. + +// repoHoldShiftGuardJS is the guard's declarations as shipped: everything +// from REPO_HOLD_SHIFT_GUARD_MS through repoHoldReleaseTooSoon. +func repoHoldShiftGuardJS(t *testing.T, html string) string { + t.Helper() + start := strings.Index(html, "const REPO_HOLD_SHIFT_GUARD_MS") + if start < 0 { + t.Fatal("index.html does not define REPO_HOLD_SHIFT_GUARD_MS") + } + last := jsFunc(t, html, "repoHoldReleaseTooSoon") + end := strings.Index(html[start:], last) + if end < 0 { + t.Fatal("repoHoldReleaseTooSoon is not declared after REPO_HOLD_SHIFT_GUARD_MS") + } + return html[start : start+end+len(last)] +} + +func runRepoHoldShiftGuardScript(t *testing.T, name, harness, assertions string) { + t.Helper() + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not on PATH — the hold shift guard was NOT executed by this run") + } + html := indexHTML(t) + start := strings.Index(html, "const HOLD_LABEL_SPELLINGS = [") + if start < 0 { + t.Fatal("index.html does not define HOLD_LABEL_SPELLINGS") + } + spellings := html[start : start+strings.Index(html[start:], "\n")] + script := "let clock = 1000000; Date.now = () => clock;\n" + + "const window = { _lastStatus: { hiveId: 'h1', repos: [{ full: 'o/r', workBreakdown: { issues: { actionable: 2, hold: 0 } }, actionableIssues: [{ number: 151, labels: [] }, { number: 283, labels: [] }], heldIssues: [], openPrs: [], heldPrs: [] }] } };\n" + + "let _statusSeqFloor = 0;\n" + + "const posts = []; const toasts = [];\n" + + "function showToast(m, k){ toasts.push(k + ':' + m); }\n" + + "function render(){}\n" + + // The real repaint path: renderRepos notes the layout on every render. + "function repaintReposForWidth(){ noteRepoHoldLayout(window._lastStatus.repos, Date.now()); }\n" + + harness + "\n" + + spellings + "\n" + + jsFunc(t, html, "canonicalHiveHoldLabel") + "\n" + + jsFunc(t, html, "holdLabels") + "\n" + + jsFunc(t, html, "repoParts") + "\n" + + jsFunc(t, html, "updateRepoBreakdownForHold") + "\n" + + jsFunc(t, html, "moveRepoItemHold") + "\n" + + repoHoldShiftGuardJS(t, html) + "\n" + + jsFunc(t, html, "toggleRepoItemHold") + "\n" + + assertions + path := filepath.Join(t.TempDir(), name+".js") + if err := os.WriteFile(path, []byte(script), 0o600); err != nil { + t.Fatal(err) + } + if out, err := exec.Command(node, path).CombinedOutput(); err != nil { + t.Fatalf("%s failed:\n%s", name, strings.TrimSpace(string(out))) + } +} + +const repoHoldPostingFetch = ` +function fetch(url, opts) { + if (opts) posts.push({ url, body: JSON.parse(opts.body) }); + return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }); +} +` + +func TestRepoHoldReleaseIgnoredWhileRowsMove(t *testing.T) { + runRepoHoldShiftGuardScript(t, "hold-shift-guard", repoHoldPostingFetch, ` +(async () => { + const fails = []; + const released = n => posts.some(p => p.url === '/api/repos/o/r/items/' + n + '/hold' && p.body.held === false); + noteRepoHoldLayout(window._lastStatus.repos, clock); // first paint + // Hold #151: its row moves into the held group. + await toggleRepoItemHold('o/r', 151, 'issue', true, { dataset: {}, textContent: '⏸ Hold' }); + // 300 ms later the click aimed at the next Hold lands on #151's Release. + clock += 300; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (released(151)) fails.push('a Release that landed while the rows moved was sent'); + if (!toasts.some(t => t.includes('o/r#151') && t.includes('Release again'))) fails.push('no toast named the unsent Release: ' + JSON.stringify(toasts)); + if (!window._lastStatus.repos[0].heldIssues.some(i => i.number === 151)) fails.push('#151 left the held group'); + // A Hold in the same window is still sent: clicking Hold down a list works. + await toggleRepoItemHold('o/r', 283, 'issue', true, { dataset: {}, textContent: '⏸ Hold' }); + if (!posts.some(p => p.url === '/api/repos/o/r/items/283/hold' && p.body.held === true)) fails.push('a Hold was delayed by the guard'); + // Once the list has been still for the guard window, Release goes through. + clock += REPO_HOLD_SHIFT_GUARD_MS; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (!released(151)) fails.push('a deliberate Release on a still list was blocked'); + if (fails.length) { console.log(fails.join('\n')); process.exit(1); } + console.log('ok'); +})().catch(e => { console.log(e && e.stack || e); process.exit(1); }); +`) +} + +func TestRepoHoldToggleOneRequestPerItemInFlight(t *testing.T) { + runRepoHoldShiftGuardScript(t, "hold-in-flight", ` +let finishFirst = null; +function fetch(url, opts) { + if (opts) { + posts.push({ url, body: JSON.parse(opts.body) }); + if (!finishFirst) return new Promise(r => { finishFirst = () => r({ ok: true, json: () => Promise.resolve({}) }); }); + } + return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }); +} +`, ` +(async () => { + const fails = []; + noteRepoHoldLayout(window._lastStatus.repos, clock); + // The optimistic repaint replaces the clicked button, so its busy mark is + // gone; a second click on the item's new Release must not race the Hold. + const first = toggleRepoItemHold('o/r', 151, 'issue', true, { dataset: {}, textContent: '⏸ Hold' }); + clock += 10 * REPO_HOLD_SHIFT_GUARD_MS; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (posts.length !== 1) fails.push('second toggle sent while the first was in flight: ' + JSON.stringify(posts)); + finishFirst(); + await first; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (posts.length !== 2 || posts[1].body.held !== false) fails.push('Release after the Hold settled was not sent: ' + JSON.stringify(posts)); + if (fails.length) { console.log(fails.join('\n')); process.exit(1); } + console.log('ok'); +})().catch(e => { console.log(e && e.stack || e); process.exit(1); }); +`) +} + +func TestRepoHoldLayoutSignatureMarksMoves(t *testing.T) { + runRepoHoldShiftGuardScript(t, "hold-layout-signature", repoHoldPostingFetch, ` +const fails = []; +const repos = [{ full: 'o/r', actionableIssues: [{ number: 1 }, { number: 2 }], heldIssues: [{ number: 3 }, { number: 4 }], openPrs: [], heldPrs: [] }]; +noteRepoHoldLayout(repos, 100); +if (repoHoldReleaseTooSoon(150)) fails.push('the first paint counted as a move'); +noteRepoHoldLayout(repos, 200); +if (repoHoldReleaseTooSoon(250)) fails.push('an unchanged repaint counted as a move'); +// A status render that only re-sorts the held group still moves the rows. +repos[0].heldIssues = [{ number: 4 }, { number: 3 }]; +noteRepoHoldLayout(repos, 300); +if (!repoHoldReleaseTooSoon(300 + REPO_HOLD_SHIFT_GUARD_MS - 1)) fails.push('a changed held group was not a move'); +if (repoHoldReleaseTooSoon(300 + REPO_HOLD_SHIFT_GUARD_MS)) fails.push('the guard outlived its window'); +if (fails.length) { console.log(fails.join('\n')); process.exit(1); } +console.log('ok'); +`) +} + +// renderRepos is what notes each layout; without the call the guard never +// sees the status render that re-sorts the card. +func TestRenderReposNotesHoldLayout(t *testing.T) { + if src := jsFunc(t, indexHTML(t), "renderRepos"); !strings.Contains(src, "noteRepoHoldLayout(repos, Date.now())") { + t.Fatal("renderRepos does not call noteRepoHoldLayout; a status re-sort would not arm the Release guard") + } +} diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index b6f16b1201..d9ad371d0e 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -2,6 +2,7 @@ refreshSectionCardShell('repos-section'); if (_repoResizeDrag || _repoOrderDrag) return; repos = repoOrderApply(repos || []); + noteRepoHoldLayout(repos, Date.now()); const el = document.getElementById('repos'); const showPlan = planningAllowed(); const showDesign = showPlan && dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); diff --git a/src/pkg/dashboard/static-src/js/repos.js b/src/pkg/dashboard/static-src/js/repos.js index 5224b72543..4926bb31c9 100644 --- a/src/pkg/dashboard/static-src/js/repos.js +++ b/src/pkg/dashboard/static-src/js/repos.js @@ -657,11 +657,49 @@ repaintReposForWidth(); return { repo, item, fromKey, toKey }; } + // A hold toggle moves its row between the held and actionable groups, and + // the status render that follows re-sorts them, so the button under the + // pointer changes a moment after every click. Clicking Hold down a card + // then landed the next click on another item's Release, and that item's + // hold came straight back off (hivecommons/hive#18054: #151 released in + // the same second #283 was held). A Release that arrives within + // REPO_HOLD_SHIFT_GUARD_MS of the rows moving is not sent; the operator + // clicks it again once the list is still. Hold is never delayed: under + // the pointer it can only park an item, and parking is what that click + // was doing anyway. + const REPO_HOLD_SHIFT_GUARD_MS = 1500; + let _repoHoldLayoutSig = null; + let _repoHoldShiftAt = 0; + const _repoHoldInFlight = {}; + // Pure: the order and membership of every card's hold-toggle rows. + function repoHoldLayoutSignature(repos) { + const nums = list => (list || []).map(x => Number(x && x.number) || 0).join(','); + return (repos || []).map(r => (r.full || r.name || '') + ':' + nums(r.actionableIssues) + '|' + nums(r.heldIssues) + '|' + nums(r.openPrs) + '|' + nums(r.heldPrs)).join(';'); + } + // Called on every repos render: remembers when the rows last moved. The + // first render is not a move; nothing was under the pointer before it. + function noteRepoHoldLayout(repos, now) { + const sig = repoHoldLayoutSignature(repos); + if (_repoHoldLayoutSig !== null && sig !== _repoHoldLayoutSig) _repoHoldShiftAt = now; + _repoHoldLayoutSig = sig; + } + function repoHoldReleaseTooSoon(now) { + return _repoHoldShiftAt > 0 && now - _repoHoldShiftAt < REPO_HOLD_SHIFT_GUARD_MS; + } async function toggleRepoItemHold(repoFull, number, type, held, btn) { if (btn && btn.dataset.busy === '1') return; const p = repoParts(repoFull); if (!p.owner || !p.repo || !number) return; const wantHeld = !!held; + // The busy mark lives on a button the optimistic repaint replaces, so + // the item itself carries the in-flight guard. + const flightKey = String(repoFull).toLowerCase() + '/' + type + '#' + number; + if (_repoHoldInFlight[flightKey]) return; + if (!wantHeld && repoHoldReleaseTooSoon(Date.now())) { + showToast('The list moved just as you clicked, so ' + repoFull + '#' + number + ' is still held. Click Release again to release it.', 'info'); + return; + } + _repoHoldInFlight[flightKey] = true; const prevText = btn ? btn.textContent : ''; const moved = moveRepoItemHold(repoFull, number, type, wantHeld, canonicalHiveHoldLabel()); if (btn) { btn.dataset.busy = '1'; btn.textContent = '…'; } @@ -680,6 +718,8 @@ if (moved) moveRepoItemHold(repoFull, number, type, !wantHeld, canonicalHiveHoldLabel()); showToast('Hold toggle failed: ' + (e && e.message ? e.message : 'network error'), 'error'); if (btn) { btn.dataset.busy = ''; btn.textContent = prevText || (wantHeld ? '⏸ Hold' : '▶ Release'); } + } finally { + delete _repoHoldInFlight[flightKey]; } } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 01999d74af..01b9fd6063 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -33934,11 +33934,49 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> repaintReposForWidth(); return { repo, item, fromKey, toKey }; } + // A hold toggle moves its row between the held and actionable groups, and + // the status render that follows re-sorts them, so the button under the + // pointer changes a moment after every click. Clicking Hold down a card + // then landed the next click on another item's Release, and that item's + // hold came straight back off (hivecommons/hive#18054: #151 released in + // the same second #283 was held). A Release that arrives within + // REPO_HOLD_SHIFT_GUARD_MS of the rows moving is not sent; the operator + // clicks it again once the list is still. Hold is never delayed: under + // the pointer it can only park an item, and parking is what that click + // was doing anyway. + const REPO_HOLD_SHIFT_GUARD_MS = 1500; + let _repoHoldLayoutSig = null; + let _repoHoldShiftAt = 0; + const _repoHoldInFlight = {}; + // Pure: the order and membership of every card's hold-toggle rows. + function repoHoldLayoutSignature(repos) { + const nums = list => (list || []).map(x => Number(x && x.number) || 0).join(','); + return (repos || []).map(r => (r.full || r.name || '') + ':' + nums(r.actionableIssues) + '|' + nums(r.heldIssues) + '|' + nums(r.openPrs) + '|' + nums(r.heldPrs)).join(';'); + } + // Called on every repos render: remembers when the rows last moved. The + // first render is not a move; nothing was under the pointer before it. + function noteRepoHoldLayout(repos, now) { + const sig = repoHoldLayoutSignature(repos); + if (_repoHoldLayoutSig !== null && sig !== _repoHoldLayoutSig) _repoHoldShiftAt = now; + _repoHoldLayoutSig = sig; + } + function repoHoldReleaseTooSoon(now) { + return _repoHoldShiftAt > 0 && now - _repoHoldShiftAt < REPO_HOLD_SHIFT_GUARD_MS; + } async function toggleRepoItemHold(repoFull, number, type, held, btn) { if (btn && btn.dataset.busy === '1') return; const p = repoParts(repoFull); if (!p.owner || !p.repo || !number) return; const wantHeld = !!held; + // The busy mark lives on a button the optimistic repaint replaces, so + // the item itself carries the in-flight guard. + const flightKey = String(repoFull).toLowerCase() + '/' + type + '#' + number; + if (_repoHoldInFlight[flightKey]) return; + if (!wantHeld && repoHoldReleaseTooSoon(Date.now())) { + showToast('The list moved just as you clicked, so ' + repoFull + '#' + number + ' is still held. Click Release again to release it.', 'info'); + return; + } + _repoHoldInFlight[flightKey] = true; const prevText = btn ? btn.textContent : ''; const moved = moveRepoItemHold(repoFull, number, type, wantHeld, canonicalHiveHoldLabel()); if (btn) { btn.dataset.busy = '1'; btn.textContent = '…'; } @@ -33957,6 +33995,8 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> if (moved) moveRepoItemHold(repoFull, number, type, !wantHeld, canonicalHiveHoldLabel()); showToast('Hold toggle failed: ' + (e && e.message ? e.message : 'network error'), 'error'); if (btn) { btn.dataset.busy = ''; btn.textContent = prevText || (wantHeld ? '⏸ Hold' : '▶ Release'); } + } finally { + delete _repoHoldInFlight[flightKey]; } } @@ -36780,6 +36820,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> refreshSectionCardShell('repos-section'); if (_repoResizeDrag || _repoOrderDrag) return; repos = repoOrderApply(repos || []); + noteRepoHoldLayout(repos, Date.now()); const el = document.getElementById('repos'); const showPlan = planningAllowed(); const showDesign = showPlan && dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); From eee2c610705da5e796d7983b31214fafa2499717 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 12:04:38 -0500 Subject: [PATCH 148/223] =?UTF-8?q?=E2=9C=A8=20hub:=20credential=20fence?= =?UTF-8?q?=20for=20self-service=20(byo-)=20hive=20heartbeats=20(#18081)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub: credential fence for self-service (byo-) hive heartbeats Step 1 of the #18060 RFC. Hive IDs in the reserved byo- namespace are listed as directory metadata only, so their heartbeat response must not carry hub-held credentials or management actions. Several delivery paths resolve what to send from spoke-reported identity (appKeySyncForHeartbeat falls back to the payload cluster_id for a hive with no SaaS record), so a per-site check is fragile. One guard in signHeartbeatResponse, the single point every heartbeat response passes through, zeroes each field that heartbeatResponseFieldFence classifies as fenced, and fails closed for any unclassified field. TestHeartbeatResponseFieldFenceInventory fails when a HeartbeatResponse field is added without a classification, following the TestNotificationWriteSitesInventory pattern. Hive-Run: hivecommons/hive#18060 Hive-Plan: RFC #18060 suggested order of work, step 1 (fence and inventory test) Hive-Spec: hivecommons/hive#18060#proposed-solution-4-credential-fence Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🚨 hub: use reflect.Pointer in self-service fence test (govet inline) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...rity-18060-selfservice-credential-fence.md | 1 + src/docs/hub-deployment.md | 4 + src/pkg/hub/heartbeat_selfservice_fence.go | 121 ++++++++++ .../hub/heartbeat_selfservice_fence_test.go | 221 ++++++++++++++++++ src/pkg/hub/heartbeat_signing.go | 10 + 5 files changed, 357 insertions(+) create mode 100644 changelog.d/security-18060-selfservice-credential-fence.md create mode 100644 src/pkg/hub/heartbeat_selfservice_fence.go create mode 100644 src/pkg/hub/heartbeat_selfservice_fence_test.go diff --git a/changelog.d/security-18060-selfservice-credential-fence.md b/changelog.d/security-18060-selfservice-credential-fence.md new file mode 100644 index 0000000000..961ff47b53 --- /dev/null +++ b/changelog.d/security-18060-selfservice-credential-fence.md @@ -0,0 +1 @@ +- The hub now withholds hub-held credentials and management actions (GitHub App config and keys, pending gateway, project config, cluster ID, access list, upgrade, restart and breaker instructions) from the heartbeat response of any hive whose ID is in the reserved `byo-` self-service namespace, through one guard driven by a classification of every response field ([#18060](https://github.com/hivecommons/hive/issues/18060)). diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index fd842d5b06..fb132e303f 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -88,6 +88,10 @@ env: Without `HIVE_HUB_SECRET`, a protected hub returns `401 unauthorized`, the spoke logs `hub heartbeat rejected status=401`, and the hub shows the hive offline. This matches the gotcha in `manual-provisioning.md`. +### Reserved `byo-` hive IDs (credential fence) + +Hive IDs that start with `byo-` are reserved for hives that their owners run and register themselves ([#18060](https://github.com/hivecommons/hive/issues/18060)). A hub lists such a hive as directory metadata only, so its heartbeat response never carries hub-held credentials or management actions, whatever key the spoke authenticated with. The hub withholds these fields: the GitHub App config (including cluster, secondary and webhook relay keys), the pending gateway, the claimed project config, the cluster ID, the authorized-user list, visibility, upgrade and branch-switch instructions, the upgrade announcement and policy, spoke restarts, agent-restart resets, fleet-breaker instructions, and self-experiment flags. Display-only data such as admin banners still flows. When the fence withholds a field, the hub logs `heartbeat: credential fence withheld fields from self-service hive` with the field names, never the values. Do not give an operator-provisioned hive a `byo-` ID. + ## Hosted spoke provisioning model The hub's SaaS provisioner creates or records: diff --git a/src/pkg/hub/heartbeat_selfservice_fence.go b/src/pkg/hub/heartbeat_selfservice_fence.go new file mode 100644 index 0000000000..191ce9d28c --- /dev/null +++ b/src/pkg/hub/heartbeat_selfservice_fence.go @@ -0,0 +1,121 @@ +package hub + +import ( + "reflect" + "strings" +) + +// Credential fence for self-service (self-hosted, owner-registered) hives +// (#18060). +// +// A self-service hive runs on its owner's infrastructure and is listed on the +// hub as directory metadata only (#12571). The hub holds no authority over its +// policy, credentials or upgrades, and every identity field such a spoke puts +// in its heartbeat payload (cluster, namespace, App installation) is reported +// by the spoke itself and cannot be trusted. Several delivery paths in the +// heartbeat response builder resolve what to send from exactly those +// payload-reported fields: appKeySyncForHeartbeat, for example, falls back to +// the spoke-reported cluster_id for a hive with no SaaS record and would hand +// that cluster's App private key to any spoke that claims it. +// +// So instead of a check at each delivery site, ONE guard runs on every +// heartbeat response before it is signed and written +// (signHeartbeatResponse is the single point every response passes through). +// The guard is driven by heartbeatResponseFieldFence, which classifies every +// HeartbeatResponse field. TestHeartbeatResponseFieldFenceInventory fails when +// a field is added to HeartbeatResponse without a classification here, so a +// new credential or management field cannot reach a self-service hive by +// default. + +// SelfServiceHiveIDPrefix is the reserved hive ID namespace for hives that +// owners register themselves (for example "byo-<github-login>-<name>"). Every +// hive ID in this namespace is fenced, whatever key it authenticated with. +const SelfServiceHiveIDPrefix = "byo-" + +// IsSelfServiceHiveID reports whether hiveID is in the reserved self-service +// namespace. +func IsSelfServiceHiveID(hiveID string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(hiveID)), SelfServiceHiveIDPrefix) +} + +// heartbeatFieldClass is how the credential fence treats one +// HeartbeatResponse field for a self-service hive. +type heartbeatFieldClass int + +const ( + // heartbeatFieldAllowed is protocol, signature or display-only data that + // carries no hub-held secret and no instruction to the spoke. + heartbeatFieldAllowed heartbeatFieldClass = iota + 1 + // heartbeatFieldFenced is a hub-held credential, a management action, hub + // policy over the hive, or data resolved from spoke-reported identity. It + // is zeroed before the response leaves the hub. + heartbeatFieldFenced +) + +// heartbeatResponseFieldFence classifies every HeartbeatResponse field by its +// Go field name. Adding a field to HeartbeatResponse requires adding it here. +var heartbeatResponseFieldFence = map[string]heartbeatFieldClass{ + // Protocol and descriptive data. + "OK": heartbeatFieldAllowed, + "HubGitHash": heartbeatFieldAllowed, + "LatestSHA": heartbeatFieldAllowed, + "LatestTag": heartbeatFieldAllowed, + "HubBanner": heartbeatFieldAllowed, // admin message, display only + "ActivityETag": heartbeatFieldAllowed, + "ActivityEventsAck": heartbeatFieldAllowed, + // Response signature binding. + "SigHiveID": heartbeatFieldAllowed, + "SigSeq": heartbeatFieldAllowed, + "SigSignedAt": heartbeatFieldAllowed, + "SigVersion": heartbeatFieldAllowed, + + // Hub-held credentials. + "GitHubAppConfig": heartbeatFieldFenced, // App, cluster and secondary keys, webhook relay key + "PendingGateway": heartbeatFieldFenced, // funded gateway API key + // Upgrade and lifecycle management. + "UpgradeTo": heartbeatFieldFenced, + "UpgradeInterrupt": heartbeatFieldFenced, + "SwitchToTag": heartbeatFieldFenced, + "RestartSpoke": heartbeatFieldFenced, + "UpgradeAnnounce": heartbeatFieldFenced, + "UpgradePolicy": heartbeatFieldFenced, // the owner, not the hub, sets upgrade posture + // Operational actions. + "ResetAgentRestarts": heartbeatFieldFenced, + "Breaker": heartbeatFieldFenced, + "SelfExperimentFlag": heartbeatFieldFenced, + // Hub policy over the hive's config and access. + "IsPublic": heartbeatFieldFenced, + "AuthorizedUsers": heartbeatFieldFenced, + "AuthorizedUserNames": heartbeatFieldFenced, + "ProjectConfig": heartbeatFieldFenced, // claimed project config + // Resolved from spoke-reported or hub-provisioned identity. + "ClusterID": heartbeatFieldFenced, +} + +// fenceSelfServiceHeartbeatResponse zeroes every fenced field of resp when +// hiveID is a self-service hive, and returns the names of fields that held a +// value (never the values). Any field missing from heartbeatResponseFieldFence +// is treated as fenced, so an unclassified field fails closed even in a build +// that skipped the inventory test. For any other hive it does nothing. +func fenceSelfServiceHeartbeatResponse(resp *HeartbeatResponse, hiveID string) []string { + if resp == nil || !IsSelfServiceHiveID(hiveID) { + return nil + } + v := reflect.ValueOf(resp).Elem() + t := v.Type() + var withheld []string + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if heartbeatResponseFieldFence[f.Name] == heartbeatFieldAllowed { + continue + } + fv := v.Field(i) + if !fv.CanSet() || fv.IsZero() { + // Unexported fields are never marshalled to the wire. + continue + } + fv.Set(reflect.Zero(f.Type)) + withheld = append(withheld, f.Name) + } + return withheld +} diff --git a/src/pkg/hub/heartbeat_selfservice_fence_test.go b/src/pkg/hub/heartbeat_selfservice_fence_test.go new file mode 100644 index 0000000000..83d7aa4be2 --- /dev/null +++ b/src/pkg/hub/heartbeat_selfservice_fence_test.go @@ -0,0 +1,221 @@ +package hub + +import ( + "reflect" + "sort" + "testing" +) + +// TestHeartbeatResponseFieldFenceInventory fails when a HeartbeatResponse +// field has no credential-fence classification, or when the fence names a +// field that no longer exists. A new field must be classified as allowed or +// fenced before it can ship (#18060). +func TestHeartbeatResponseFieldFenceInventory(t *testing.T) { + typ := reflect.TypeOf(HeartbeatResponse{}) + seen := map[string]bool{} + var missing []string + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + seen[name] = true + switch heartbeatResponseFieldFence[name] { + case heartbeatFieldAllowed, heartbeatFieldFenced: + default: + missing = append(missing, name) + } + } + if len(missing) > 0 { + sort.Strings(missing) + t.Errorf("HeartbeatResponse fields with no credential-fence classification: %v\n"+ + "add each to heartbeatResponseFieldFence in heartbeat_selfservice_fence.go as "+ + "heartbeatFieldAllowed (protocol/display only) or heartbeatFieldFenced "+ + "(credential, management action, hub policy, or spoke-reported identity)", missing) + } + var stale []string + for name := range heartbeatResponseFieldFence { + if !seen[name] { + stale = append(stale, name) + } + } + if len(stale) > 0 { + sort.Strings(stale) + t.Errorf("heartbeatResponseFieldFence names fields HeartbeatResponse no longer has: %v", stale) + } +} + +// The fields the RFC names explicitly must stay fenced, so a later edit to the +// table cannot quietly reclassify a credential as allowed. +func TestHeartbeatResponseFieldFenceCoversCredentialsAndManagement(t *testing.T) { + for _, name := range []string{ + "GitHubAppConfig", "PendingGateway", "ProjectConfig", "ClusterID", + "UpgradeTo", "UpgradeInterrupt", "SwitchToTag", "UpgradeAnnounce", + "RestartSpoke", "AuthorizedUsers", "AuthorizedUserNames", + } { + if heartbeatResponseFieldFence[name] != heartbeatFieldFenced { + t.Errorf("%s must be heartbeatFieldFenced", name) + } + } +} + +func TestIsSelfServiceHiveID(t *testing.T) { + cases := map[string]bool{ + "byo-alice-lab": true, + "BYO-alice-lab": true, + " byo-x": true, + "byo": false, + "hosted-acme": false, + "saas-acme": false, + "kelly-headwaters": false, + "my-byo-hive": false, + "": false, + "byoverlay-cluster": false, + } + for id, want := range cases { + if got := IsSelfServiceHiveID(id); got != want { + t.Errorf("IsSelfServiceHiveID(%q) = %v, want %v", id, got, want) + } + } +} + +// filledHeartbeatResponse sets every exported field of a HeartbeatResponse to +// a non-zero value, so the fence test covers fields added after it was written. +func filledHeartbeatResponse(t *testing.T) *HeartbeatResponse { + t.Helper() + resp := &HeartbeatResponse{} + v := reflect.ValueOf(resp).Elem() + for i := 0; i < v.NumField(); i++ { + fv := v.Field(i) + if !fv.CanSet() { + continue + } + fillNonZero(t, v.Type().Field(i).Name, fv) + if fv.IsZero() { + t.Fatalf("could not give HeartbeatResponse.%s a non-zero value", v.Type().Field(i).Name) + } + } + return resp +} + +func fillNonZero(t *testing.T, name string, fv reflect.Value) { + t.Helper() + switch fv.Kind() { + case reflect.Bool: + fv.SetBool(true) + case reflect.String: + fv.SetString("x") + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + fv.SetInt(1) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + fv.SetUint(1) + case reflect.Slice: + fv.Set(reflect.MakeSlice(fv.Type(), 1, 1)) + case reflect.Map: + fv.Set(reflect.MakeMap(fv.Type())) + fv.SetMapIndex(reflect.Zero(fv.Type().Key()), reflect.Zero(fv.Type().Elem())) + case reflect.Pointer: + fv.Set(reflect.New(fv.Type().Elem())) + case reflect.Struct: + for i := 0; i < fv.NumField(); i++ { + if fv.Field(i).CanSet() { + fillNonZero(t, name, fv.Field(i)) + return + } + } + default: + t.Fatalf("fillNonZero: unhandled kind %s for HeartbeatResponse.%s", fv.Kind(), name) + } +} + +func TestFenceSelfServiceHeartbeatResponseZeroesFencedFields(t *testing.T) { + resp := filledHeartbeatResponse(t) + withheld := fenceSelfServiceHeartbeatResponse(resp, "byo-alice-lab") + + withheldSet := map[string]bool{} + for _, name := range withheld { + withheldSet[name] = true + } + v := reflect.ValueOf(resp).Elem() + for i := 0; i < v.NumField(); i++ { + name := v.Type().Field(i).Name + switch heartbeatResponseFieldFence[name] { + case heartbeatFieldAllowed: + if v.Field(i).IsZero() { + t.Errorf("allowed field %s was cleared", name) + } + if withheldSet[name] { + t.Errorf("allowed field %s reported as withheld", name) + } + default: + if !v.Field(i).IsZero() { + t.Errorf("fenced field %s still set for a self-service hive", name) + } + if !withheldSet[name] { + t.Errorf("fenced field %s cleared but not reported as withheld", name) + } + } + } +} + +func TestFenceSelfServiceHeartbeatResponseLeavesOtherHivesAlone(t *testing.T) { + for _, id := range []string{"hosted-acme", "kelly-headwaters", ""} { + resp := filledHeartbeatResponse(t) + want := *resp + if withheld := fenceSelfServiceHeartbeatResponse(resp, id); withheld != nil { + t.Errorf("hive %q: withheld %v, want nothing", id, withheld) + } + if !reflect.DeepEqual(*resp, want) { + t.Errorf("hive %q: response changed by the fence", id) + } + } + if fenceSelfServiceHeartbeatResponse(nil, "byo-x") != nil { + t.Error("nil response must be a no-op") + } +} + +// End to end through the heartbeat handler: hub-held credentials queued for +// a self-service hive ID never reach the wire, while the same queue for an +// operator-listed hive still delivers, and display-only data still flows. +func TestHeartbeatHandlerFencesSelfServiceHive(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + + queue := func(s *HubServer, id string) { + s.pendingGitHubAppConfigs[id] = &HeartbeatGitHubAppConfig{AppID: 42, InstallationID: 7, PrivateKey: "-----BEGIN KEY-----"} + s.pendingGateways[id] = &HeartbeatGatewayConfig{Name: "openrouter", Kind: "openrouter", Endpoint: "https://example.invalid", Key: "sk-secret"} + s.hubBanners[id] = &HubBannerEntry{ID: "b1", Message: "maintenance tonight", Color: "info"} + } + + t.Run("self-service hive gets no credentials", func(t *testing.T) { + s := newHeartbeatHub() + const id = "byo-alice-lab" + queue(s, id) + resp := decodeHeartbeatResponse(t, s, `{"hive_id":"`+id+`","org":"alice","cluster_id":"hive-oke"}`) + if resp.GitHubAppConfig != nil { + t.Errorf("GitHubAppConfig delivered to a self-service hive: %+v", resp.GitHubAppConfig) + } + if resp.PendingGateway != nil { + t.Errorf("PendingGateway delivered to a self-service hive: %+v", resp.PendingGateway) + } + if resp.UpgradePolicy != nil { + t.Errorf("UpgradePolicy delivered to a self-service hive: %+v", resp.UpgradePolicy) + } + if !resp.OK || resp.HubGitHash == "" { + t.Errorf("protocol fields lost: ok=%v hub_git_hash=%q", resp.OK, resp.HubGitHash) + } + if resp.HubBanner == nil || resp.HubBanner.Message != "maintenance tonight" { + t.Errorf("display-only banner not delivered: %+v", resp.HubBanner) + } + }) + + t.Run("operator-listed hive still gets them", func(t *testing.T) { + s := newHeartbeatHub() + const id = "alice-lab" + queue(s, id) + resp := decodeHeartbeatResponse(t, s, `{"hive_id":"`+id+`","org":"alice","cluster_id":"hive-oke"}`) + if resp.GitHubAppConfig == nil || resp.GitHubAppConfig.PrivateKey == "" { + t.Errorf("GitHubAppConfig not delivered to an operator-listed hive: %+v", resp.GitHubAppConfig) + } + if resp.PendingGateway == nil { + t.Error("PendingGateway not delivered to an operator-listed hive") + } + }) +} diff --git a/src/pkg/hub/heartbeat_signing.go b/src/pkg/hub/heartbeat_signing.go index 8329642c27..94691f7362 100644 --- a/src/pkg/hub/heartbeat_signing.go +++ b/src/pkg/hub/heartbeat_signing.go @@ -3,6 +3,7 @@ package hub import ( "encoding/json" "net/http" + "strings" "sync" "time" @@ -54,7 +55,16 @@ func (h *heartbeatSeqState) next(hiveID string, now time.Time) int64 { // and it marshals resp once here to sign the exact bytes; the caller marshals // again to write. The two marshals are byte-identical (same struct, same // process), so the signature covers what the spoke receives. +// +// It also applies the self-service credential fence (#18060) first, and +// before the keyless early return, because every heartbeat response passes +// through here on its way to the wire: a self-service hive never receives a +// fenced field, signed or not. func (s *HubServer) signHeartbeatResponse(w http.ResponseWriter, resp *HeartbeatResponse, hiveID string) { + if withheld := fenceSelfServiceHeartbeatResponse(resp, hiveID); len(withheld) > 0 && s.logger != nil { + s.logger.Warn("heartbeat: credential fence withheld fields from self-service hive", + "hive_id", hiveID, "fields", strings.Join(withheld, ",")) + } seed := s.ssoSigningSeed() if seed == "" || hiveID == "" { // No key or no identity: leave the response unsigned (legacy behaviour). From 42036ceea420e83b9a8eb6d04e007cd9199e49ee Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 12:12:36 -0500 Subject: [PATCH 149/223] =?UTF-8?q?=F0=9F=90=9B=20knowledge:=20persist=20d?= =?UTF-8?q?ashboard-imported=20documents=20so=20upgrades=20keep=20them=20(?= =?UTF-8?q?#18088)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git sources connected from the dashboard are written to knowledge.git_sources and re-connected at boot, but an imported document (PDF, HTML, DOCX, text) had no record outside its /data/knowledge/documents/<slug>/metadata.json. When that on-disk record did not survive an upgrade, the document vanished from the knowledge sources list while the GitHub sources stayed. Record dashboard imports in knowledge.documents (deduped by slug), drop the entry on delete, and switch the boot loop to EnsureDocument, which keeps a document already reloaded from disk and re-imports a configured one that is missing. This also stops the boot loop from logging an "already imported" warning for every configured document on restart. Fixes #18058 Hive-Run: hivecommons/hive#18058 Hive-Plan: issue-18058-document-source-persistence Hive-Spec: hive#18058-upgrade-keeps-knowledge-sources Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: nate-double-u <4453979+nate-double-u@users.noreply.github.com> --- ...-18058-document-sources-survive-upgrade.md | 1 + src/cmd/hive/main.go | 10 ++- src/pkg/dashboard/api_knowledge.go | 53 ++++++++++++ .../api_knowledge_documents_persist_test.go | 82 +++++++++++++++++++ src/pkg/knowledge/api.go | 20 +++++ src/pkg/knowledge/api_ensure_document_test.go | 64 +++++++++++++++ 6 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-18058-document-sources-survive-upgrade.md create mode 100644 src/pkg/dashboard/api_knowledge_documents_persist_test.go create mode 100644 src/pkg/knowledge/api_ensure_document_test.go diff --git a/changelog.d/fixed-18058-document-sources-survive-upgrade.md b/changelog.d/fixed-18058-document-sources-survive-upgrade.md new file mode 100644 index 0000000000..473e1afa0a --- /dev/null +++ b/changelog.d/fixed-18058-document-sources-survive-upgrade.md @@ -0,0 +1 @@ +- Documents imported from the dashboard (PDF, HTML, DOCX, text) are now saved to `knowledge.documents` in `hive.yaml` like git sources, and boot re-imports any configured document whose on-disk record is missing, so an upgrade no longer drops a PDF knowledge source while keeping GitHub ones. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 68a979c6e3..4f5841fb67 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3125,12 +3125,20 @@ func (b *boot) bootKnowledgeWith(deps bootKnowledgeDeps) { FilePath: doc.FilePath, Layer: knowledge.LayerType(doc.Layer), } - meta, err := b.knowledgeAPI.ImportDocument(b.ctx, docConfig) + // EnsureDocument keeps a document already reloaded from disk and + // re-imports one whose on-disk state is gone, so a configured + // document survives an upgrade the way git sources do (#18058). + meta, imported, err := b.knowledgeAPI.EnsureDocument(b.ctx, docConfig) if err != nil { b.logger.Warn("failed to import document source", "name", doc.Name, "error", err, ) + } else if !imported { + b.logger.Debug("document source already loaded", + "name", doc.Name, + "facts", meta.FactCount, + ) } else { b.logger.Info("document source imported", "name", doc.Name, diff --git a/src/pkg/dashboard/api_knowledge.go b/src/pkg/dashboard/api_knowledge.go index f0643d563d..b3414363cf 100644 --- a/src/pkg/dashboard/api_knowledge.go +++ b/src/pkg/dashboard/api_knowledge.go @@ -1283,10 +1283,62 @@ func (s *Server) handleDocumentsImport(w http.ResponseWriter, r *http.Request) { jsonError(w, err.Error(), http.StatusInternalServerError) return } + s.persistDocumentSource(req) s.auditFromRequest(r, "document_import", auditDetail("name", req.Name), "") jsonResponse(w, meta) } +// persistDocumentSource records a dashboard-imported document in +// knowledge.documents, as handleGitSourcesConnect does for git sources. The +// document's metadata.json on the data volume was its only record, so an +// upgrade that lost it dropped the document while config-backed git sources +// survived (#18058); boot re-imports configured documents that are missing. +// Context7 imports are not recorded: knowledge.documents has no context7_id +// and Context7 has its own auto_import list. +func (s *Server) persistDocumentSource(req knowledge.DocSourceConfig) { + if s.deps.Config == nil || req.Context7ID != "" { + return + } + slug := knowledge.DocumentSlug(req.Name) + for _, d := range s.deps.Config.Knowledge.Documents { + if knowledge.DocumentSlug(d.Name) == slug { + return + } + } + s.deps.Config.Knowledge.Documents = append(s.deps.Config.Knowledge.Documents, config.DocSourceConfigYAML{ + Name: req.Name, + URL: req.URL, + FilePath: req.FilePath, + Layer: string(req.Layer), + }) + if err := s.saveConfig(); err != nil { + s.logger.Error("failed to persist config after document import", "error", err) + } +} + +// unpersistDocumentSource drops a deleted document from knowledge.documents +// so boot does not re-import it. +func (s *Server) unpersistDocumentSource(slug string) { + if s.deps.Config == nil { + return + } + docs := s.deps.Config.Knowledge.Documents + filtered := make([]config.DocSourceConfigYAML, 0, len(docs)) + for _, d := range docs { + if knowledge.DocumentSlug(d.Name) == slug { + continue + } + filtered = append(filtered, d) + } + if len(filtered) == len(docs) { + return + } + s.deps.Config.Knowledge.Documents = filtered + if err := s.saveConfig(); err != nil { + s.logger.Error("failed to persist config after document delete", "error", err) + } +} + func (s *Server) handleDocumentGet(w http.ResponseWriter, r *http.Request) { if !s.ensureKnowledge() { jsonError(w, "knowledge not configured", http.StatusServiceUnavailable) @@ -1313,6 +1365,7 @@ func (s *Server) handleDocumentDelete(w http.ResponseWriter, r *http.Request) { jsonError(w, err.Error(), http.StatusNotFound) return } + s.unpersistDocumentSource(slug) s.auditFromRequest(r, "document_delete", auditDetail("slug", slug), "") okResponse(w, map[string]string{"status": "deleted", "slug": slug}) } diff --git a/src/pkg/dashboard/api_knowledge_documents_persist_test.go b/src/pkg/dashboard/api_knowledge_documents_persist_test.go new file mode 100644 index 0000000000..247335c2f2 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_documents_persist_test.go @@ -0,0 +1,82 @@ +package dashboard + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge" +) + +// TestDocumentsImportPersistsAndDeleteUnpersists pins #18058: a document +// imported from the dashboard is recorded in knowledge.documents (as git +// sources are in knowledge.git_sources) so boot can restore it after an +// upgrade, a repeat import does not duplicate the entry, and deleting the +// document drops it from config so boot does not resurrect it. +func TestDocumentsImportPersistsAndDeleteUnpersists(t *testing.T) { + base := t.TempDir() + knowledge.SetBaseDirForTest(t, base) + source := filepath.Join(base, "uploads", "guide.txt") + if err := os.MkdirAll(filepath.Dir(source), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("Guide\n\nAgents must cite the tech docs guide."), 0o644); err != nil { + t.Fatal(err) + } + + s, deps := apiServer(t) + deps.Config.SourcePath = filepath.Join(t.TempDir(), "hive.yaml") + deps.Knowledge = knowledge.NewKnowledgeAPI(nil, knowledge.KnowledgeConfig{Enabled: true, Engine: "file"}, deps.Logger) + deps.Config.Knowledge.Documents = []config.DocSourceConfigYAML{{Name: "other-doc", URL: "https://example.com/other.pdf", Layer: "project"}} + + rec := doPost(s, "/api/knowledge/documents", map[string]any{"name": "Tech Docs Guide", "file_path": source}) + if rec.Code != http.StatusOK { + t.Fatalf("import: expected 200, got %d (body %q)", rec.Code, rec.Body.String()) + } + want := []config.DocSourceConfigYAML{ + {Name: "other-doc", URL: "https://example.com/other.pdf", Layer: "project"}, + {Name: "Tech Docs Guide", FilePath: source, Layer: "project"}, + } + assertDocs := func(stage string, want []config.DocSourceConfigYAML) { + t.Helper() + got := deps.Config.Knowledge.Documents + if len(got) != len(want) { + t.Fatalf("%s: knowledge.documents = %+v, want %+v", stage, got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("%s: knowledge.documents = %+v, want %+v", stage, got, want) + } + } + } + assertDocs("after import", want) + persisted, err := os.ReadFile(deps.Config.SourcePath) + if err != nil { + t.Fatalf("read persisted config: %v", err) + } + if !strings.Contains(string(persisted), "Tech Docs Guide") { + t.Fatalf("persisted config is missing the imported document:\n%s", persisted) + } + + // A config entry for the same slug (e.g. a re-import after delete-from-disk) + // must not be duplicated; the knowledge API rejects the second import, so + // exercise the dedupe directly. + s.persistDocumentSource(knowledge.DocSourceConfig{Name: "tech docs guide", FilePath: source, Layer: knowledge.LayerProject}) + assertDocs("after duplicate persist", want) + + rec = doDelete(s, "/api/knowledge/documents/"+knowledge.DocumentSlug("Tech Docs Guide"), nil) + if rec.Code != http.StatusOK { + t.Fatalf("delete: expected 200, got %d (body %q)", rec.Code, rec.Body.String()) + } + assertDocs("after delete", want[:1]) + persisted, err = os.ReadFile(deps.Config.SourcePath) + if err != nil { + t.Fatalf("read persisted config: %v", err) + } + if text := string(persisted); strings.Contains(text, "Tech Docs Guide") || !strings.Contains(text, "other-doc") { + t.Fatalf("persisted config did not drop only the deleted document:\n%s", text) + } +} diff --git a/src/pkg/knowledge/api.go b/src/pkg/knowledge/api.go index c83db1ea61..d26886445e 100644 --- a/src/pkg/knowledge/api.go +++ b/src/pkg/knowledge/api.go @@ -1105,6 +1105,26 @@ func (k *KnowledgeAPI) ImportDocument(ctx context.Context, config DocSourceConfi return meta, nil } +// DocumentSlug returns the slug a document named name is stored under, the +// same key GetDocument and DeleteDocument take. +func DocumentSlug(name string) string { return slugify(name) } + +// EnsureDocument imports config unless a document with the same slug is +// already loaded (typically reloaded from its on-disk metadata at startup). +// imported reports whether a fresh import ran. It is the boot path for +// knowledge.documents: a configured document whose on-disk state was lost is +// re-imported instead of silently disappearing (#18058). +func (k *KnowledgeAPI) EnsureDocument(ctx context.Context, config DocSourceConfig) (meta *DocMetadata, imported bool, err error) { + if existing, getErr := k.GetDocument(DocumentSlug(config.Name)); getErr == nil { + return existing, false, nil + } + meta, err = k.ImportDocument(ctx, config) + if err != nil { + return nil, false, err + } + return meta, true, nil +} + // ListDocuments returns metadata for all imported documents. func (k *KnowledgeAPI) ListDocuments() []DocMetadata { k.mu.RLock() diff --git a/src/pkg/knowledge/api_ensure_document_test.go b/src/pkg/knowledge/api_ensure_document_test.go new file mode 100644 index 0000000000..1f276f68cb --- /dev/null +++ b/src/pkg/knowledge/api_ensure_document_test.go @@ -0,0 +1,64 @@ +package knowledge + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// TestEnsureDocumentReimportsAfterLostDiskState pins #18058: a configured +// document whose on-disk metadata is gone after an upgrade must be imported +// again at boot rather than silently dropped, while one that was reloaded +// from disk must not be imported a second time. +func TestEnsureDocumentReimportsAfterLostDiskState(t *testing.T) { + base := withKnowledgeBaseDir(t) + source := filepath.Join(base, "uploads", "guide.txt") + if err := os.MkdirAll(filepath.Dir(source), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("Guide\n\nAgents must cite the tech docs guide."), 0o644); err != nil { + t.Fatal(err) + } + cfg := DocSourceConfig{Name: "Tech Docs Guide", FilePath: source, Layer: LayerProject} + + first := NewKnowledgeAPI(nil, KnowledgeConfig{Enabled: true}, importDocTestLogger()) + meta, imported, err := first.EnsureDocument(context.Background(), cfg) + if err != nil || !imported { + t.Fatalf("first EnsureDocument = %+v, imported=%v, err=%v; want a fresh import", meta, imported, err) + } + slug := DocumentSlug(cfg.Name) + if meta.Slug != slug { + t.Fatalf("slug = %q, want %q", meta.Slug, slug) + } + + // Restart with the disk state intact: the reloaded document is kept as-is. + restarted := NewKnowledgeAPI(nil, KnowledgeConfig{Enabled: true}, importDocTestLogger()) + got, imported, err := restarted.EnsureDocument(context.Background(), cfg) + if err != nil { + t.Fatalf("EnsureDocument after restart: %v", err) + } + if imported { + t.Fatal("EnsureDocument re-imported a document that was reloaded from disk") + } + if got.Slug != slug || got.FactCount != meta.FactCount { + t.Fatalf("EnsureDocument after restart = %+v, want reloaded %+v", got, meta) + } + + // Upgrade lost the document's on-disk record: the configured source must + // come back instead of vanishing from the knowledge sources list. + if err := os.RemoveAll(filepath.Join(base, docStorageDir)); err != nil { + t.Fatal(err) + } + upgraded := NewKnowledgeAPI(nil, KnowledgeConfig{Enabled: true}, importDocTestLogger()) + if docs := upgraded.ListDocuments(); len(docs) != 0 { + t.Fatalf("precondition: expected no documents reloaded, got %+v", docs) + } + got, imported, err = upgraded.EnsureDocument(context.Background(), cfg) + if err != nil || !imported { + t.Fatalf("EnsureDocument after lost disk state = %+v, imported=%v, err=%v; want a re-import", got, imported, err) + } + if docs := upgraded.ListDocuments(); len(docs) != 1 || docs[0].Slug != slug { + t.Fatalf("ListDocuments after re-import = %+v, want %q", docs, slug) + } +} From de6d2fb871c26847d604769f8160295bd7a8118f Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 17:15:23 +0000 Subject: [PATCH 150/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.242.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.242.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 16 ++++++++++++++++ changelog.d/added-chevron-stage-icons.md | 1 - changelog.d/changed-18063.md | 1 - changelog.d/changed-18074.md | 1 - .../fixed-benchmarks-canonical-model-id.md | 1 - changelog.d/fixed-role-claim-reject-log.md | 1 - 6 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-chevron-stage-icons.md delete mode 100644 changelog.d/changed-18063.md delete mode 100644 changelog.d/changed-18074.md delete mode 100644 changelog.d/fixed-benchmarks-canonical-model-id.md delete mode 100644 changelog.d/fixed-role-claim-reject-log.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5290f196c0..79c94c0d44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.242.0) + +### Added + +- Activity monitor: a workflow-model chevron that is not the row's active stage now wears a glyph for what the stage means (inbox, bolt, hand, pencil, pull-request, flask, eye, people, thumbs-up, merge, check, …) picked by the column's name, dimmed to 60%, instead of its clipped label; the active chevron leads its word with the same glyph. Workflow columns take an optional `icon` (an allowlisted icon id or a single emoji, validated server- and client-side; never markup), and the workflow editor's column inspector previews the default and lets the operator replace it (#18033). + +### Changed + +- images: bump gh 2.103.3 -> 2.104.0 (#18063) +- kicks: register REFERENCE REPOS as layout section 10a and golden its wording (#18074) + +### Fixed + +- Benchmarks export now adds `model.canonical` and `model.effort` next to the reported id and derives `model.provider` from the model family, so `claude-sonnet-5-5` (Copilot CLI), `claude-sonnet-5.5` (Claude Code) and `openai-codex/gpt-6.1-sol` no longer appear as separate models on hivecommons.dev/benchmarks. +- Repeated trust-tier agent role claim rejections over contribute-ws are now logged once per user and role at INFO instead of flooding WARN, so they no longer surface as unclassified self-reported errors (#18071). + ## 2026-11-13 (v5.241.1) ### Changed diff --git a/changelog.d/added-chevron-stage-icons.md b/changelog.d/added-chevron-stage-icons.md deleted file mode 100644 index 341298d6e9..0000000000 --- a/changelog.d/added-chevron-stage-icons.md +++ /dev/null @@ -1 +0,0 @@ -- Activity monitor: a workflow-model chevron that is not the row's active stage now wears a glyph for what the stage means (inbox, bolt, hand, pencil, pull-request, flask, eye, people, thumbs-up, merge, check, …) picked by the column's name, dimmed to 60%, instead of its clipped label; the active chevron leads its word with the same glyph. Workflow columns take an optional `icon` (an allowlisted icon id or a single emoji, validated server- and client-side; never markup), and the workflow editor's column inspector previews the default and lets the operator replace it (#18033). diff --git a/changelog.d/changed-18063.md b/changelog.d/changed-18063.md deleted file mode 100644 index 8aa025ec82..0000000000 --- a/changelog.d/changed-18063.md +++ /dev/null @@ -1 +0,0 @@ -- images: bump gh 2.103.3 -> 2.104.0 (#18063) diff --git a/changelog.d/changed-18074.md b/changelog.d/changed-18074.md deleted file mode 100644 index beeb0a4829..0000000000 --- a/changelog.d/changed-18074.md +++ /dev/null @@ -1 +0,0 @@ -- kicks: register REFERENCE REPOS as layout section 10a and golden its wording (#18074) diff --git a/changelog.d/fixed-benchmarks-canonical-model-id.md b/changelog.d/fixed-benchmarks-canonical-model-id.md deleted file mode 100644 index 044a0042fc..0000000000 --- a/changelog.d/fixed-benchmarks-canonical-model-id.md +++ /dev/null @@ -1 +0,0 @@ -- Benchmarks export now adds `model.canonical` and `model.effort` next to the reported id and derives `model.provider` from the model family, so `claude-sonnet-5-5` (Copilot CLI), `claude-sonnet-5.5` (Claude Code) and `openai-codex/gpt-6.1-sol` no longer appear as separate models on hivecommons.dev/benchmarks. diff --git a/changelog.d/fixed-role-claim-reject-log.md b/changelog.d/fixed-role-claim-reject-log.md deleted file mode 100644 index e5ca0de6d8..0000000000 --- a/changelog.d/fixed-role-claim-reject-log.md +++ /dev/null @@ -1 +0,0 @@ -- Repeated trust-tier agent role claim rejections over contribute-ws are now logged once per user and role at INFO instead of flooding WARN, so they no longer surface as unclassified self-reported errors (#18071). From aa568ed1d1ad66e9b2f0b59cc8c87fac67a259d0 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 12:42:44 -0500 Subject: [PATCH 151/223] fix(dashboard): surface Check result for knowledge connectors (#18096) The Knowledge Integrations Check button showed a 'checking' toast and then nothing. kbPlaudAuthLoad now returns the state it stored and kbPlaudRefreshAuth reports it: an error toast with the server's error string verbatim (e.g. the 404 'no knowledge connectors are configured'), or a success/info toast with the sign-in state. The card still re-renders. Fixes #18092 Signed-off-by: Andy Anderson <andy@clubanderson.com> --- .../fixed-18092-knowledge-check-toast.md | 1 + .../knowledge_plaud_check_feedback.test.js | 96 +++++++++++++++++++ src/pkg/dashboard/static-src/js/knowledge.js | 14 ++- src/pkg/dashboard/static/index.html | 14 ++- 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-18092-knowledge-check-toast.md create mode 100644 src/pkg/dashboard/knowledge_plaud_check_feedback.test.js diff --git a/changelog.d/fixed-18092-knowledge-check-toast.md b/changelog.d/fixed-18092-knowledge-check-toast.md new file mode 100644 index 0000000000..1500f367c0 --- /dev/null +++ b/changelog.d/fixed-18092-knowledge-check-toast.md @@ -0,0 +1 @@ +- Knowledge Integrations: the per-connector **Check** button now reports its result — a success/info toast with the sign-in state (and the card refreshed), or an error toast carrying the server's `error` string verbatim (for example `no knowledge connectors are configured`) instead of a "checking" toast followed by silence ([#18092](https://github.com/hivecommons/hive/issues/18092)). diff --git a/src/pkg/dashboard/knowledge_plaud_check_feedback.test.js b/src/pkg/dashboard/knowledge_plaud_check_feedback.test.js new file mode 100644 index 0000000000..26aa1006a3 --- /dev/null +++ b/src/pkg/dashboard/knowledge_plaud_check_feedback.test.js @@ -0,0 +1,96 @@ +// Knowledge Integrations "Check" (hivecommons/hive#18092): the per-connector +// Check button must always report an outcome. A failed status call surfaces +// the server's `error` string verbatim in an error toast; a successful one +// surfaces the sign-in state in a success/info toast and re-renders the card. +// Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +// Like the other dashboard tests' fnSrc, but keeps a leading `async` so the +// extracted handlers can still await. +const ASYNC_PREFIX = 'async '; +const fnSrc = name => { + let start = html.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + if (html.slice(start - ASYNC_PREFIX.length, start) === ASYNC_PREFIX) start -= ASYNC_PREFIX.length; + let depth = 0; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) return html.slice(start, i + 1); + } + throw new Error('unbalanced ' + name); +}; + +const NOT_CONFIGURED = 'no knowledge connectors are configured'; +const HTTP_NOT_FOUND = 404; +const HTTP_OK = 200; + +const setup = ({ status, body, cardPresent = true }) => { + const toasts = []; + const calls = { urls: [], renders: 0 }; + const fetch = async url => { + calls.urls.push(url); + return { ok: status === HTTP_OK, status, json: async () => body }; + }; + const document = { querySelector: () => (cardPresent ? {} : null) }; + const CSS = { escape: s => s }; + const h = new Function('fetch', 'document', 'CSS', 'showToast', 'kbRenderIntegrationsModal', + 'const _kbPlaudAuth = {};\n' + fnSrc('kbPlaudAuthLoad') + '\n' + fnSrc('kbPlaudAuthSummary') + '\n' + fnSrc('kbPlaudRefreshAuth') + + '\nreturn { kbPlaudRefreshAuth, _kbPlaudAuth };')(fetch, document, CSS, (m, k) => toasts.push([m, k]), () => { calls.renders++; }); + return { h, toasts, calls }; +}; + +test('a 404 from the auth endpoint shows an error toast with the server error verbatim', async () => { + const { h, toasts, calls } = setup({ status: HTTP_NOT_FOUND, body: { error: NOT_CONFIGURED } }); + await h.kbPlaudRefreshAuth('plaud-voice'); + assert.deepStrictEqual(calls.urls, ['/api/knowledge/connectors/plaud-voice/auth?refresh=1']); + assert.strictEqual(toasts[0][1], 'info'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'error'); + assert.ok(last[0].includes(NOT_CONFIGURED), 'toast must carry the server error: ' + last[0]); + assert.strictEqual(calls.renders, 1, 'card re-rendered with the error state'); + assert.strictEqual(h._kbPlaudAuth['plaud-voice'].error, NOT_CONFIGURED); +}); + +test('a non-JSON failure falls back to the HTTP status in the error toast', async () => { + const { h, toasts } = setup({ status: HTTP_NOT_FOUND, body: {} }); + await h.kbPlaudRefreshAuth('p'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'error'); + assert.ok(last[0].includes('HTTP ' + HTTP_NOT_FOUND), last[0]); +}); + +test('a connected state shows a success toast naming the account and refreshes the card', async () => { + const { h, toasts, calls } = setup({ status: HTTP_OK, body: { connected: true, account: 'andy' } }); + await h.kbPlaudRefreshAuth('plaud-voice'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'success'); + assert.ok(last[0].includes('plaud-voice') && last[0].includes('connected as andy'), last[0]); + assert.strictEqual(calls.renders, 1); +}); + +test('a not-connected state is reported as info, with the server message', async () => { + const { h, toasts } = setup({ status: HTTP_OK, body: { connected: false, message: 'token expired' } }); + await h.kbPlaudRefreshAuth('p'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'info'); + assert.ok(last[0].includes('not connected') && last[0].includes('token expired'), last[0]); +}); + +test('a pending state tells the user where the authorization link is', async () => { + const { h, toasts } = setup({ status: HTTP_OK, body: { pending: true, auth_url: 'https://example.invalid/auth' } }); + await h.kbPlaudRefreshAuth('p'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'info'); + assert.ok(last[0].includes('waiting for authorization'), last[0]); +}); + +test('the toast is still shown when the card is no longer in the DOM', async () => { + const { h, toasts, calls } = setup({ status: HTTP_NOT_FOUND, body: { error: NOT_CONFIGURED }, cardPresent: false }); + await h.kbPlaudRefreshAuth('p'); + assert.strictEqual(calls.renders, 0); + assert.strictEqual(toasts[toasts.length - 1][1], 'error'); +}); diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index ac6380b949..4cd1dfa12c 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2596,10 +2596,22 @@ _kbPlaudAuth[name] = { error: e.message }; } if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + return _kbPlaudAuth[name]; } + // One-line summary of a sign-in state for the Check toast (hivecommons/hive#18092). + function kbPlaudAuthSummary(a) { + if (a.connected) return 'connected' + (a.account ? ' as ' + a.account : ''); + if (a.pending) return 'waiting for authorization' + (a.auth_url ? ' — open the link on the card' : ''); + return 'not connected' + (a.message ? ' — ' + a.message : ''); + } + // Check: re-query the sign-in state and always report the outcome, so a + // 404 "no knowledge connectors are configured" reads as an error toast + // with the server's message verbatim, not as silence (hivecommons/hive#18092). async function kbPlaudRefreshAuth(name) { showToast('Checking Plaud sign-in…', 'info'); - await kbPlaudAuthLoad(name, true); + const a = (await kbPlaudAuthLoad(name, true)) || {}; + if (a.error) showToast('Check failed: ' + a.error, 'error'); + else showToast('Plaud ' + name + ': ' + kbPlaudAuthSummary(a), a.connected ? 'success' : 'info'); } async function kbPlaudConnect(name) { try { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 01b9fd6063..a78345f735 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46526,10 +46526,22 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> _kbPlaudAuth[name] = { error: e.message }; } if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + return _kbPlaudAuth[name]; } + // One-line summary of a sign-in state for the Check toast (hivecommons/hive#18092). + function kbPlaudAuthSummary(a) { + if (a.connected) return 'connected' + (a.account ? ' as ' + a.account : ''); + if (a.pending) return 'waiting for authorization' + (a.auth_url ? ' — open the link on the card' : ''); + return 'not connected' + (a.message ? ' — ' + a.message : ''); + } + // Check: re-query the sign-in state and always report the outcome, so a + // 404 "no knowledge connectors are configured" reads as an error toast + // with the server's message verbatim, not as silence (hivecommons/hive#18092). async function kbPlaudRefreshAuth(name) { showToast('Checking Plaud sign-in…', 'info'); - await kbPlaudAuthLoad(name, true); + const a = (await kbPlaudAuthLoad(name, true)) || {}; + if (a.error) showToast('Check failed: ' + a.error, 'error'); + else showToast('Plaud ' + name + ': ' + kbPlaudAuthSummary(a), a.connected ? 'success' : 'info'); } async function kbPlaudConnect(name) { try { From bafc68a34e6934a889dd2742308bcabb2bc0096f Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 12:45:06 -0500 Subject: [PATCH 152/223] =?UTF-8?q?=E2=9C=A8=20Link=20the=20fleet=20benchm?= =?UTF-8?q?arks=20from=20the=20hub=20nav,=20My=20Hives=20and=20the=20spoke?= =?UTF-8?q?=20FAQ=20(#18039)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Link the fleet benchmarks from the hub nav, My Hives and the spoke FAQ Fixes #18034 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 benchmarks links: use tokens instead of inline styles Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 dashboard: Twitch link test tolerates text after the FAQ link The FAQ intro now continues with a benchmarks sentence, so the test no longer pins the closing </p> right after the Twitch link. Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-benchmarks-links.md | 1 + src/pkg/dashboard/static-src/html/faq.html | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/dashboard/twitch_link_test.go | 2 +- src/pkg/hub/assets/dashboard.html | 4 ++++ src/pkg/hub/static/index.html | 1 + src/pkg/hub/static/my-hives.html | 1 + 7 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-benchmarks-links.md diff --git a/changelog.d/added-benchmarks-links.md b/changelog.d/added-benchmarks-links.md new file mode 100644 index 0000000000..26cf95b2bb --- /dev/null +++ b/changelog.d/added-benchmarks-links.md @@ -0,0 +1 @@ +- Hub header nav and My Hives now link to the fleet benchmarks page (hivecommons.dev/benchmarks), and the spoke dashboard FAQ points at it for model comparisons (#18034). diff --git a/src/pkg/dashboard/static-src/html/faq.html b/src/pkg/dashboard/static-src/html/faq.html index 6075f22dea..41ace5bf37 100644 --- a/src/pkg/dashboard/static-src/html/faq.html +++ b/src/pkg/dashboard/static-src/html/faq.html @@ -6,7 +6,7 @@ <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" <p class="faq-intro">Short answers to the questions people ask once a hive is running. Everything below describes what this hive actually does today — where behaviour is surprising, it is called out rather than smoothed over. Each answer links to the source doc, so this panel stays a map, not a second source of truth.</p> - <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>.</p> + <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>. Wondering which model to run? <a class="text-accent fw-semibold" href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener">Compare models on real fleet PR outcomes</a> (merge rate, CI first-try, estimated cost).</p> <div class="faq-filter-row"> <label for="faq-filter" class="faq-filter-label">Filter</label> diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index a78345f735..af856dff8b 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -9198,7 +9198,7 @@ <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" <p class="faq-intro">Short answers to the questions people ask once a hive is running. Everything below describes what this hive actually does today — where behaviour is surprising, it is called out rather than smoothed over. Each answer links to the source doc, so this panel stays a map, not a second source of truth.</p> - <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>.</p> + <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>. Wondering which model to run? <a class="text-accent fw-semibold" href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener">Compare models on real fleet PR outcomes</a> (merge rate, CI first-try, estimated cost).</p> <div class="faq-filter-row"> <label for="faq-filter" class="faq-filter-label">Filter</label> diff --git a/src/pkg/dashboard/twitch_link_test.go b/src/pkg/dashboard/twitch_link_test.go index 0ad4f8aa70..d5d462a828 100644 --- a/src/pkg/dashboard/twitch_link_test.go +++ b/src/pkg/dashboard/twitch_link_test.go @@ -12,7 +12,7 @@ func TestTwitchLiveStreamLinkInDashboardShell(t *testing.T) { for _, want := range []string{ `<a class="oc-nav-item" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener"`, `<span class="oc-nav-text">Watch live on Twitch</span>`, - `Watch the hive live on Twitch</a>.</p>`, + `Watch the hive live on Twitch</a>.`, } { if !strings.Contains(html, want) { t.Fatalf("index.html missing %q", want) diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 2e910c1e5e..c1a2da62b5 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -107,6 +107,8 @@ } h1 { letter-spacing: 0; font-size: clamp(2rem, 4vw, 3rem); line-height: .98; margin-bottom: .8rem; } .subtitle { color: var(--muted); font-size: 1.02rem; line-height: 1.7; margin-bottom: var(--sp-9); } + .benchmarks-pointer { margin-top: var(--sp-3); padding: var(--sp-3) var(--sp-5); border: 1px solid var(--line); border-radius: var(--r-sm); font-size: var(--fs-sm); color: var(--muted); } + .benchmarks-pointer a { color: var(--amber); font-weight: 700; } /* ── Table ── */ /* Status filter chips above the My Hives table. --chip-color is set inline @@ -764,6 +766,7 @@ <a href="/fleet" id="nav-fleet" style="display:none" title="Fleet health — agents the governor expects on but that can't work">Fleet</a> <a href="/api/docs" target="_blank">API</a> <a href="https://kubestellar.io/docs/hive/overview/introduction" target="_blank" rel="noopener">Docs</a> + <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener" title="Real PR outcomes per model and agent across the fleet">Benchmarks</a> <span id="nav-user" class="nav-user"></span> </nav> <div class="header-right"> @@ -779,6 +782,7 @@ <h1>My Hives</h1> <p class="subtitle">Hive instances you own or have access to</p> <p style="margin-top:var(--sp-4);padding:8px 14px;border:1px solid var(--line);border-radius:8px;background:color-mix(in srgb,var(--hub-dashboard-accent) 8%,transparent);font-size:0.85rem">👋 New to Hive? Read the <a href="https://docs.kubestellar.io/docs/hive/getting-started" target="_blank" rel="noopener" style="color:var(--amber);font-weight:700">Getting Started Guide</a> before diving in.</p> + <p id="benchmarks-pointer" class="benchmarks-pointer">📊 See how models perform across the fleet — merge rate, CI first-try and estimated cost per PR, from real hive output. <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener">Open the benchmarks ↗</a></p> <p id="latest-image-sha" style="font-size:0.7rem;color:var(--muted);margin-top:var(--sp-2)"></p> <!-- Image-pulls bar chart: per-release container-image PULLS of the public spoke image (ghcr.io/hivecommons/hive), bucketed by the diff --git a/src/pkg/hub/static/index.html b/src/pkg/hub/static/index.html index 09c9211c9e..b423cf2a99 100644 --- a/src/pkg/hub/static/index.html +++ b/src/pkg/hub/static/index.html @@ -623,6 +623,7 @@ <a href="#how-it-works">How it works</a> <a href="/learn">Learn</a> <a href="https://docs.hivecommons.dev" target="_blank" rel="noopener noreferrer">Docs</a> + <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener noreferrer" title="Real PR outcomes per model and agent across the fleet">Benchmarks</a> <a href="/reading">Reading</a> <a href="/get-started">Get Started</a> <a href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a> diff --git a/src/pkg/hub/static/my-hives.html b/src/pkg/hub/static/my-hives.html index 5f66da02e0..23b2345128 100644 --- a/src/pkg/hub/static/my-hives.html +++ b/src/pkg/hub/static/my-hives.html @@ -242,6 +242,7 @@ <a href="/fleet" id="nav-fleet" class="active" aria-current="page" title="Fleet health — agents the governor expects on but that can't work">Fleet</a> <a href="/api/docs">API</a> <a href="https://docs.hivecommons.dev" target="_blank" rel="noopener">Docs</a> + <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener" title="Real PR outcomes per model and agent across the fleet">Benchmarks</a> <span id="nav-user" class="nav-user"></span> </nav> <div class="header-right"> From 0c46b61d8820905a0c687bc405049697fa878a2a Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 17:48:05 +0000 Subject: [PATCH 153/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.243.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.243.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 20 +++++++++++++++++++ changelog.d/added-18021-reference-tier.md | 1 - ...changed-18028-webhook-create-idempotent.md | 1 - ...anged-review-perspectives-source-reload.md | 1 - changelog.d/fixed-18054-hold-release-guard.md | 1 - ...-18058-document-sources-survive-upgrade.md | 1 - ...rity-18060-selfservice-credential-fence.md | 1 - 7 files changed, 20 insertions(+), 6 deletions(-) delete mode 100644 changelog.d/added-18021-reference-tier.md delete mode 100644 changelog.d/changed-18028-webhook-create-idempotent.md delete mode 100644 changelog.d/changed-review-perspectives-source-reload.md delete mode 100644 changelog.d/fixed-18054-hold-release-guard.md delete mode 100644 changelog.d/fixed-18058-document-sources-survive-upgrade.md delete mode 100644 changelog.d/security-18060-selfservice-credential-fence.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c94c0d44..72f6020238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,26 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.243.0) + +### Added + +- `/api/status` `referenceRepos[]` entries now carry an additive `reference_tier` field (`read-only`, `issues`, `prs` or `issues+prs`) so API consumers can tell a `project.reference_repos` entry from a managed repo; the entries stay out of `repos[]` and carry no ACMM, work-item or pause fields, and the dashboard's `📚 reference · <tier>` chip reads it ([#18021](https://github.com/hivecommons/hive/issues/18021)). + +### Changed + +- Dashboard: "Create webhook for me" is idempotent — it returns the existing hook for this hive's payload URL instead of creating a duplicate, and answers 412 naming the missing `organization_hooks`/`repository_hooks` write permission (#18028) +- **`review.perspectives_source` is now resolved at startup and on every config reload, and the review relay perspective set is reinstalled each time**, so a verdict naming a perspective the link removed is refused and one it added is accepted without a restart. A dashboard edit of the review config no longer drops the linked set, and a malformed or unreachable document keeps the last-known-good set with a warning. ([#17988](https://github.com/hivecommons/hive/issues/17988), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) + +### Fixed + +- dashboard: a Projects-card Release that lands while the rows are re-sorting after a Hold is no longer sent, so holding items down a list no longer releases a neighbouring item's hold (#18054) +- Documents imported from the dashboard (PDF, HTML, DOCX, text) are now saved to `knowledge.documents` in `hive.yaml` like git sources, and boot re-imports any configured document whose on-disk record is missing, so an upgrade no longer drops a PDF knowledge source while keeping GitHub ones. + +### Security + +- The hub now withholds hub-held credentials and management actions (GitHub App config and keys, pending gateway, project config, cluster ID, access list, upgrade, restart and breaker instructions) from the heartbeat response of any hive whose ID is in the reserved `byo-` self-service namespace, through one guard driven by a classification of every response field ([#18060](https://github.com/hivecommons/hive/issues/18060)). + ## 2026-11-13 (v5.242.0) ### Added diff --git a/changelog.d/added-18021-reference-tier.md b/changelog.d/added-18021-reference-tier.md deleted file mode 100644 index 358b39b129..0000000000 --- a/changelog.d/added-18021-reference-tier.md +++ /dev/null @@ -1 +0,0 @@ -- `/api/status` `referenceRepos[]` entries now carry an additive `reference_tier` field (`read-only`, `issues`, `prs` or `issues+prs`) so API consumers can tell a `project.reference_repos` entry from a managed repo; the entries stay out of `repos[]` and carry no ACMM, work-item or pause fields, and the dashboard's `📚 reference · <tier>` chip reads it ([#18021](https://github.com/hivecommons/hive/issues/18021)). diff --git a/changelog.d/changed-18028-webhook-create-idempotent.md b/changelog.d/changed-18028-webhook-create-idempotent.md deleted file mode 100644 index 01d2c011f1..0000000000 --- a/changelog.d/changed-18028-webhook-create-idempotent.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: "Create webhook for me" is idempotent — it returns the existing hook for this hive's payload URL instead of creating a duplicate, and answers 412 naming the missing `organization_hooks`/`repository_hooks` write permission (#18028) diff --git a/changelog.d/changed-review-perspectives-source-reload.md b/changelog.d/changed-review-perspectives-source-reload.md deleted file mode 100644 index c517a8ac1f..0000000000 --- a/changelog.d/changed-review-perspectives-source-reload.md +++ /dev/null @@ -1 +0,0 @@ -- **`review.perspectives_source` is now resolved at startup and on every config reload, and the review relay perspective set is reinstalled each time**, so a verdict naming a perspective the link removed is refused and one it added is accepted without a restart. A dashboard edit of the review config no longer drops the linked set, and a malformed or unreachable document keeps the last-known-good set with a warning. ([#17988](https://github.com/hivecommons/hive/issues/17988), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-18054-hold-release-guard.md b/changelog.d/fixed-18054-hold-release-guard.md deleted file mode 100644 index 6e43c24352..0000000000 --- a/changelog.d/fixed-18054-hold-release-guard.md +++ /dev/null @@ -1 +0,0 @@ -- dashboard: a Projects-card Release that lands while the rows are re-sorting after a Hold is no longer sent, so holding items down a list no longer releases a neighbouring item's hold (#18054) diff --git a/changelog.d/fixed-18058-document-sources-survive-upgrade.md b/changelog.d/fixed-18058-document-sources-survive-upgrade.md deleted file mode 100644 index 473e1afa0a..0000000000 --- a/changelog.d/fixed-18058-document-sources-survive-upgrade.md +++ /dev/null @@ -1 +0,0 @@ -- Documents imported from the dashboard (PDF, HTML, DOCX, text) are now saved to `knowledge.documents` in `hive.yaml` like git sources, and boot re-imports any configured document whose on-disk record is missing, so an upgrade no longer drops a PDF knowledge source while keeping GitHub ones. diff --git a/changelog.d/security-18060-selfservice-credential-fence.md b/changelog.d/security-18060-selfservice-credential-fence.md deleted file mode 100644 index 961ff47b53..0000000000 --- a/changelog.d/security-18060-selfservice-credential-fence.md +++ /dev/null @@ -1 +0,0 @@ -- The hub now withholds hub-held credentials and management actions (GitHub App config and keys, pending gateway, project config, cluster ID, access list, upgrade, restart and breaker instructions) from the heartbeat response of any hive whose ID is in the reserved `byo-` self-service namespace, through one guard driven by a classification of every response field ([#18060](https://github.com/hivecommons/hive/issues/18060)). From 1a326c6150cdc92c7f726e9d5df8eb143eb12592 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 12:59:45 -0500 Subject: [PATCH 154/223] =?UTF-8?q?=F0=9F=90=9B=20knowledge:=20'not=20load?= =?UTF-8?q?ed'=20badge=20and=20disabled=20actions=20when=20the=20connector?= =?UTF-8?q?=20runtime=20is=20not=20running=20(#18097)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dashboard): show 'not loaded' badge when the connector runtime is not running When the status response carries runtime:false (hive booted with no knowledge.connectors, so the syncer is nil), connector cards showed a 'pending' badge, which reads as 'awaiting OAuth sign-in'. Cards now show a distinct 'not loaded' badge with a one-line restart hint and docs link, disable Connect / Check / Sync with the same hint instead of letting them 404, and skip the Plaud auth probe that would also 404. 'pending' remains reserved for a loaded connector awaiting sign-in or first sync. Fixes #18093 Signed-off-by: Andy Anderson <andy@clubanderson.com> * fix(dashboard): only an explicit runtime:false marks a connector not loaded A failed or absent /status response was coerced to runtime:false, which would have rendered every card as 'not loaded' with a restart instruction after a transient 5xx. Store st.runtime !== false instead, drop the unused kc-not-loaded class, and cover the failed-status, missing-key and explicit false cases in the test. Signed-off-by: Andy Anderson <andy@clubanderson.com> * test(dashboard): match the Plaud card button call after the not-loaded change kbPlaudAuthButtons now takes a disabledAttr argument, so the index.html snippet asserted by TestKnowledgeIntegrationsModalPlaudCard is updated to the new call shape; the auth-line snippet gains its else prefix. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...93-knowledge-connector-not-loaded-badge.md | 1 + .../knowledge_connector_not_loaded.test.js | 132 ++++++++++++++++++ .../knowledge_integrations_modal_test.go | 4 +- .../dashboard/static-src/css/knowledge.css | 1 + src/pkg/dashboard/static-src/js/knowledge.js | 37 +++-- src/pkg/dashboard/static/index.html | 38 +++-- 6 files changed, 193 insertions(+), 20 deletions(-) create mode 100644 changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md create mode 100644 src/pkg/dashboard/knowledge_connector_not_loaded.test.js diff --git a/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md b/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md new file mode 100644 index 0000000000..319a50e047 --- /dev/null +++ b/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md @@ -0,0 +1 @@ +- Knowledge Integrations: when the connector syncer is not running (`runtime: false`, hive booted with no `knowledge.connectors`), connector cards now show a `not loaded` badge with a restart hint and docs link instead of `pending`, and Connect / Check / Sync are disabled with the same hint rather than failing with 404 toasts; `pending` stays reserved for a loaded connector awaiting sign-in ([#18093](https://github.com/hivecommons/hive/issues/18093)). diff --git a/src/pkg/dashboard/knowledge_connector_not_loaded.test.js b/src/pkg/dashboard/knowledge_connector_not_loaded.test.js new file mode 100644 index 0000000000..b04bd8cc41 --- /dev/null +++ b/src/pkg/dashboard/knowledge_connector_not_loaded.test.js @@ -0,0 +1,132 @@ +// Knowledge Integrations connector card when the backend syncer is not running +// (hivecommons/hive#18093): the status response's `runtime: false` turns the +// `pending` badge into a distinct `not loaded` badge with a one-line hint and +// a docs link, and disables Connect / Check / Sync with the same hint. A loaded +// connector awaiting sign-in keeps `pending`; a disabled entry keeps `disabled`. +// Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +// Keeps a leading `async` so extracted loaders can still await. +const ASYNC_PREFIX = 'async '; +const fnSrc = name => { + let start = html.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + if (html.slice(start - ASYNC_PREFIX.length, start) === ASYNC_PREFIX) start -= ASYNC_PREFIX.length; + let depth = 0; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) return html.slice(start, i + 1); + } + throw new Error('unbalanced ' + name); +}; +// The two string constants next to the helpers, lifted from the source so the +// test does not duplicate the badge text or hint. +const constSrc = name => { + const m = html.match(new RegExp('const ' + name + " = '([^']*)';")); + assert.ok(m, name + ' not found'); + return m[1]; +}; + +const NOT_LOADED = constSrc('KC_STATUS_NOT_LOADED'); +const HINT = constSrc('KC_NOT_LOADED_HINT'); +const OWNER = true; + +const render = ({ runtime, c, row, type = 'plaud', owner = OWNER }) => { + const calls = { authLines: 0 }; + const escapeHtml = s => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); + const h = new Function('_kcState', '_kbPlaudAuth', 'escapeHtml', 'kcAgo', 'kbPlaudAuthLine', 'kbPlaudPasteHtml', 'kbWisprFlowSourceLine', + fnSrc('kbConnectorNotLoaded') + '\n' + fnSrc('kbConnectorNotLoadedHtml') + '\n' + fnSrc('kbPlaudAuthButtons') + '\n' + fnSrc('kbConnectorRow') + + "\nconst KC_STATUS_NOT_LOADED = '" + NOT_LOADED + "'; const KC_NOT_LOADED_HINT = '" + HINT + "';" + + '\nreturn kbConnectorRow;')( + { runtime, rows: row ? [row] : [], publish: null }, {}, escapeHtml, () => '—', + () => { calls.authLines++; return '<div data-kb-plaud-auth></div>'; }, () => '', () => ''); + return { html: h(type, c, owner), calls }; +}; + +const badge = out => { const m = out.match(/<span class="kc-pill" data-status="([^"]*)">([^<]*)<\/span>/); assert.ok(m, 'badge missing'); assert.strictEqual(m[1], m[2]); return m[1]; }; +const buttons = out => [...out.matchAll(/<button [^>]*data-action="([^"]+)"[^>]*>/g)].map(m => ({ action: m[1], disabled: / disabled\b/.test(m[0]), title: (m[0].match(/title="([^"]*)"/) || [])[1] })); + +test('runtime:false shows a not-loaded badge, the hint and a docs link instead of pending', () => { + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + const { html: out, calls } = render({ runtime: false, c, row: { name: 'plaud-voice', status: 'pending' } }); + assert.strictEqual(badge(out), NOT_LOADED); + assert.ok(!out.includes('>pending<'), 'pending badge must not appear'); + assert.ok(out.includes('data-kb-not-loaded'), 'hint line rendered'); + assert.ok(out.includes(HINT), 'hint text rendered'); + assert.ok(out.includes('href="/docs/knowledge-connectors.md"'), 'docs link rendered'); + assert.strictEqual(calls.authLines, 0, 'the Plaud auth probe (which would 404) is not rendered'); +}); + +test('runtime:false disables Connect, Check and Sync and explains why in the title', () => { + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + const { html: out } = render({ runtime: false, c, row: null }); + const b = buttons(out); + const byAction = Object.fromEntries(b.map(x => [x.action, x])); + for (const action of ['kbPlaudConnect', 'kbPlaudRefreshAuth', 'kbIntegrationSync']) { + assert.ok(byAction[action], action + ' rendered'); + assert.ok(byAction[action].disabled, action + ' disabled'); + assert.strictEqual(byAction[action].title, HINT, action + ' carries the hint'); + } + assert.ok(!byAction.kbIntegrationEdit.disabled, 'Edit stays enabled'); + assert.ok(!byAction.kbIntegrationToggle.disabled, 'Enable/Disable stays enabled'); +}); + +test('runtime:true keeps pending for a loaded connector awaiting sign-in, with live buttons', () => { + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + const { html: out, calls } = render({ runtime: true, c, row: { name: 'plaud-voice', status: 'pending' } }); + assert.strictEqual(badge(out), 'pending'); + assert.ok(!out.includes('data-kb-not-loaded')); + assert.strictEqual(calls.authLines, 1); + const b = buttons(out); + assert.ok(b.some(x => x.action === 'kbPlaudRefreshAuth' && !x.disabled), 'Check enabled'); + assert.ok(b.some(x => x.action === 'kbIntegrationSync' && !x.disabled), 'Sync enabled'); +}); + +test('a disabled entry keeps the disabled badge even when the runtime is not loaded', () => { + const c = { name: 'wiki', type: 'confluence', enabled: false }; + const { html: out } = render({ runtime: false, c, row: { name: 'wiki', status: 'disabled' }, type: 'confluence' }); + assert.strictEqual(badge(out), 'disabled'); + assert.ok(!out.includes('data-kb-not-loaded')); +}); + +// Drive the real loader with stubbed config + status fetches and return the +// runtime flag it stores, so the status-fetch failure path is covered. +const HTTP_OK = 200; +const HTTP_SERVER_ERROR = 500; +const loadRuntime = async statusRes => { + const cfgRes = { ok: true, status: HTTP_OK, json: async () => ({ connectors: [{ name: 'plaud-voice', type: 'plaud', enabled: true }], types: ['plaud'] }) }; + const fetch = async url => (url.endsWith('/status') ? statusRes : cfgRes); + const state = { layers: [] }; + const run = new Function('fetch', '_kcState', 'kbRenderIntegrationsModal', + "let _kbIntegrationSelected = 'plaud';\n" + fnSrc('kbLoadIntegrationsModal') + '\nreturn kbLoadIntegrationsModal("plaud").then(() => _kcState);'); + return (await run(fetch, state, () => {})).runtime; +}; + +test('a failed status fetch leaves runtime true, so cards keep pending rather than claiming not loaded', async () => { + assert.strictEqual(await loadRuntime({ ok: false, status: HTTP_SERVER_ERROR, json: async () => ({ error: 'boom' }) }), true); + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + assert.strictEqual(badge(render({ runtime: true, c, row: null }).html), 'pending'); +}); + +test('a status response without a runtime key is treated as loaded', async () => { + assert.strictEqual(await loadRuntime({ ok: true, status: HTTP_OK, json: async () => ({ connectors: [] }) }), true); +}); + +test('an explicit runtime:false in the status response is stored and renders not loaded', async () => { + const runtime = await loadRuntime({ ok: true, status: HTTP_OK, json: async () => ({ connectors: [], runtime: false }) }); + assert.strictEqual(runtime, false); + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + assert.strictEqual(badge(render({ runtime, c, row: null }).html), NOT_LOADED); +}); + +test('non-owner view still shows the not-loaded badge and hint without action buttons', () => { + const c = { name: 'wiki', type: 'confluence', enabled: true }; + const { html: out } = render({ runtime: false, c, row: null, type: 'confluence', owner: false }); + assert.strictEqual(badge(out), NOT_LOADED); + assert.ok(out.includes('data-kb-not-loaded')); + assert.strictEqual(buttons(out).length, 0); +}); diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 8dc4184b81..c10e188d90 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -155,10 +155,10 @@ func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { for _, want := range []string{ `'plaud': { label: 'Plaud voice notes'`, `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, - `if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, + `else if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, `data-action="kbPlaudPasteComplete"`, `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete'`, - `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, + `if (type === 'plaud') html += kbPlaudAuthButtons(c.name, notLoaded ? disabledAttr : '');`, `data-action="kbPlaudConnect"`, `data-action="kbPlaudDisconnect"`, `data-action="kbPlaudRefreshAuth"`, diff --git a/src/pkg/dashboard/static-src/css/knowledge.css b/src/pkg/dashboard/static-src/css/knowledge.css index 1f129c0545..8c4ff2e085 100644 --- a/src/pkg/dashboard/static-src/css/knowledge.css +++ b/src/pkg/dashboard/static-src/css/knowledge.css @@ -18,6 +18,7 @@ .kc-pill[data-status="ok"] { background: var(--green); } .kc-pill[data-status="syncing"] { background: var(--blue); } .kc-pill[data-status="error"] { background: var(--red); } + .kc-pill[data-status="not loaded"] { background: var(--orange); } .kc-publish { margin-top: var(--sp-4); border-top: var(--line-width) solid var(--border); padding-top: var(--sp-3); } .kc-publish-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: var(--sp-2); } .kc-publish-head h4 { margin: 0; } diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 4cd1dfa12c..2438327e5c 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2367,7 +2367,7 @@ const cfg = await cfgRes.json().catch(() => ({})); if (!cfgRes.ok) throw new Error(cfg.error || ('HTTP ' + cfgRes.status)); const st = stRes.ok ? await stRes.json().catch(() => ({})) : {}; - _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: !!st.runtime, publish: st.publish || null }); + _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: st.runtime !== false, publish: st.publish || null }); } catch (e) { _kcState = Object.assign(_kcState, { loaded: true, error: e.message }); } @@ -2423,23 +2423,40 @@ return body; } + // Badge shown when the hive's connector syncer is not running (status + // response `runtime: false`): the entry is saved but this process never + // constructed it, so sign-in, Check and Sync would all 404. Distinct from + // `pending`, which is a loaded connector awaiting sign-in or first sync. + // Only an explicit `runtime: false` counts: a failed or absent status + // response must not tell the owner to restart the hive + // (hivecommons/hive#18093). + const KC_STATUS_NOT_LOADED = 'not loaded'; + const KC_NOT_LOADED_HINT = 'Connector runtime is not running on this hive — restart the hive after enabling knowledge connectors.'; + function kbConnectorNotLoaded(c) { return !!c.enabled && !_kcState.runtime; } + function kbConnectorNotLoadedHtml() { + return '<div class="muted-xs" data-kb-not-loaded>' + escapeHtml(KC_NOT_LOADED_HINT) + ' <a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener">Docs ↗</a></div>'; + } + function kbConnectorRow(type, c, owner) { const byName = {}; (_kcState.rows || []).forEach(r => { byName[r.name] = r; }); const r = byName[c.name] || {}; - const status = r.status || (c.enabled ? 'pending' : 'disabled'); + const notLoaded = kbConnectorNotLoaded(c); + const status = notLoaded ? KC_STATUS_NOT_LOADED : (r.status || (c.enabled ? 'pending' : 'disabled')); const publish = _kcState.publish && _kcState.publish.connector === c.name ? ' · publish ' + (_kcState.publish.running ? 'syncing' : _kcState.publish.last_error ? 'error' : _kcState.publish.last_success ? 'ok' : 'pending') : ''; let html = '<div class="card-inset row-between mb-3" data-kb-connector-name="' + escapeHtml(c.name) + '"><div class="kb-integration-row-main">'; html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); + if (notLoaded) html += kbConnectorNotLoadedHtml(); + else if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { + const disabledAttr = ' disabled title="' + escapeHtml(KC_NOT_LOADED_HINT) + '"'; html += '<div class="kb-integration-actions">'; - if (type === 'plaud') html += kbPlaudAuthButtons(c.name); - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name, notLoaded ? disabledAttr : ''); + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (notLoaded ? disabledAttr : status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationEdit" data-arg0="' + escapeHtml(c.name) + '">Edit</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationToggle" data-arg0="' + escapeHtml(c.name) + '">' + (c.enabled ? 'Disable' : 'Enable') + '</button>'; html += '</div>'; @@ -2577,13 +2594,15 @@ else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; } - function kbPlaudAuthButtons(name) { + // disabledAttr, when set, is appended to every button (connector not loaded, #18093). + function kbPlaudAuthButtons(name, disabledAttr) { const a = _kbPlaudAuth[name] || {}; const n = escapeHtml(name); + const off = disabledAttr || ''; let html = ''; - if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '">Disconnect</button>'; - else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (a.pending ? ' disabled' : '') + '>Connect Plaud</button>'; - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '" title="Re-check sign-in with Plaud">Check</button>'; + if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '"' + off + '>Disconnect</button>'; + else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (off || (a.pending ? ' disabled' : '')) + '>Connect Plaud</button>'; + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '"' + (off || ' title="Re-check sign-in with Plaud"') + '>Check</button>'; return html; } async function kbPlaudAuthLoad(name, refresh) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index af856dff8b..6359a27634 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5821,6 +5821,7 @@ .kc-pill[data-status="ok"] { background: var(--green); } .kc-pill[data-status="syncing"] { background: var(--blue); } .kc-pill[data-status="error"] { background: var(--red); } + .kc-pill[data-status="not loaded"] { background: var(--orange); } .kc-publish { margin-top: var(--sp-4); border-top: var(--line-width) solid var(--border); padding-top: var(--sp-3); } .kc-publish-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: var(--sp-2); } .kc-publish-head h4 { margin: 0; } @@ -46297,7 +46298,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const cfg = await cfgRes.json().catch(() => ({})); if (!cfgRes.ok) throw new Error(cfg.error || ('HTTP ' + cfgRes.status)); const st = stRes.ok ? await stRes.json().catch(() => ({})) : {}; - _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: !!st.runtime, publish: st.publish || null }); + _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: st.runtime !== false, publish: st.publish || null }); } catch (e) { _kcState = Object.assign(_kcState, { loaded: true, error: e.message }); } @@ -46353,23 +46354,40 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> return body; } + // Badge shown when the hive's connector syncer is not running (status + // response `runtime: false`): the entry is saved but this process never + // constructed it, so sign-in, Check and Sync would all 404. Distinct from + // `pending`, which is a loaded connector awaiting sign-in or first sync. + // Only an explicit `runtime: false` counts: a failed or absent status + // response must not tell the owner to restart the hive + // (hivecommons/hive#18093). + const KC_STATUS_NOT_LOADED = 'not loaded'; + const KC_NOT_LOADED_HINT = 'Connector runtime is not running on this hive — restart the hive after enabling knowledge connectors.'; + function kbConnectorNotLoaded(c) { return !!c.enabled && !_kcState.runtime; } + function kbConnectorNotLoadedHtml() { + return '<div class="muted-xs" data-kb-not-loaded>' + escapeHtml(KC_NOT_LOADED_HINT) + ' <a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener">Docs ↗</a></div>'; + } + function kbConnectorRow(type, c, owner) { const byName = {}; (_kcState.rows || []).forEach(r => { byName[r.name] = r; }); const r = byName[c.name] || {}; - const status = r.status || (c.enabled ? 'pending' : 'disabled'); + const notLoaded = kbConnectorNotLoaded(c); + const status = notLoaded ? KC_STATUS_NOT_LOADED : (r.status || (c.enabled ? 'pending' : 'disabled')); const publish = _kcState.publish && _kcState.publish.connector === c.name ? ' · publish ' + (_kcState.publish.running ? 'syncing' : _kcState.publish.last_error ? 'error' : _kcState.publish.last_success ? 'ok' : 'pending') : ''; let html = '<div class="card-inset row-between mb-3" data-kb-connector-name="' + escapeHtml(c.name) + '"><div class="kb-integration-row-main">'; html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); + if (notLoaded) html += kbConnectorNotLoadedHtml(); + else if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { + const disabledAttr = ' disabled title="' + escapeHtml(KC_NOT_LOADED_HINT) + '"'; html += '<div class="kb-integration-actions">'; - if (type === 'plaud') html += kbPlaudAuthButtons(c.name); - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name, notLoaded ? disabledAttr : ''); + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (notLoaded ? disabledAttr : status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationEdit" data-arg0="' + escapeHtml(c.name) + '">Edit</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationToggle" data-arg0="' + escapeHtml(c.name) + '">' + (c.enabled ? 'Disable' : 'Enable') + '</button>'; html += '</div>'; @@ -46507,13 +46525,15 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; } - function kbPlaudAuthButtons(name) { + // disabledAttr, when set, is appended to every button (connector not loaded, #18093). + function kbPlaudAuthButtons(name, disabledAttr) { const a = _kbPlaudAuth[name] || {}; const n = escapeHtml(name); + const off = disabledAttr || ''; let html = ''; - if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '">Disconnect</button>'; - else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (a.pending ? ' disabled' : '') + '>Connect Plaud</button>'; - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '" title="Re-check sign-in with Plaud">Check</button>'; + if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '"' + off + '>Disconnect</button>'; + else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (off || (a.pending ? ' disabled' : '')) + '>Connect Plaud</button>'; + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '"' + (off || ' title="Re-check sign-in with Plaud"') + '>Check</button>'; return html; } async function kbPlaudAuthLoad(name, refresh) { From 7e160aeb3310f86b768b2887c5e6b06cd5f36605 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:09:44 -0500 Subject: [PATCH 155/223] =?UTF-8?q?=F0=9F=90=9B=20fix(ioscan):=20retry=20o?= =?UTF-8?q?ne=20transient=20classifier=20endpoint=20failure=20before=20fai?= =?UTF-8?q?ling=20open=20(#18106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single 429/5xx or dropped connection from the ioscan model endpoint made the scheduler apply its fail mode immediately: under fail-open the text passed on unclassified and the hive filed an ioscan_classifier_unavailable selfreport (#18105, previously #15077 on the same fingerprint). The HTTP chat client now retries such a transient failure once after 500ms. Timeouts and permanent statuses (4xx other than 429) are not retried, and a caller cancellation during the pause returns the original error. Hive-Run: hivecommons/hive#18105 Hive-Plan: selfreport ioscan_classifier_unavailable fix Hive-Spec: ioscan-classifier#fail-mode Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .../fixed-18105-ioscan-classifier-retry.md | 1 + src/docs/fleet-report.md | 2 +- src/pkg/ioscan/classifier.go | 80 ++++++- src/pkg/ioscan/classifier_retry_test.go | 208 ++++++++++++++++++ 4 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-18105-ioscan-classifier-retry.md create mode 100644 src/pkg/ioscan/classifier_retry_test.go diff --git a/changelog.d/fixed-18105-ioscan-classifier-retry.md b/changelog.d/fixed-18105-ioscan-classifier-retry.md new file mode 100644 index 0000000000..133036f418 --- /dev/null +++ b/changelog.d/fixed-18105-ioscan-classifier-retry.md @@ -0,0 +1 @@ +- The ioscan classifier now retries a single transient model-endpoint failure (HTTP 429/5xx or a dropped connection) before applying its fail mode, so one gateway blip no longer passes untrusted text on unclassified or withholds it ([#18105](https://github.com/hivecommons/hive/issues/18105)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 7c0b9a20dc..f0b350b191 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -150,7 +150,7 @@ title is `[selfreport] <class>: <short message>`. | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | -| `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | +| `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). The HTTP model endpoint retries one transient failure (429, 500/502/503/504, or a connection dropped before a reply; not a timeout) before either fail mode applies, so a single gateway blip is neither withheld nor passed on unclassified. Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/pkg/ioscan/classifier.go b/src/pkg/ioscan/classifier.go index c09b20591f..523980f8e7 100644 --- a/src/pkg/ioscan/classifier.go +++ b/src/pkg/ioscan/classifier.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -27,6 +28,7 @@ const ( DefaultClassifierCacheEntries = 1024 DefaultClassifierBudgetPerEval = 64 classifierTimeout = 20 * time.Second + classifierRetryDelay = 500 * time.Millisecond classifierMaxTokens = 180 classifierMaxRationaleChars = 500 SemanticClassifierRule = "injection.semantic_classifier" @@ -202,10 +204,11 @@ func NewLLMClassifier(c LLMClassifierConfig) (*LLMClassifier, error) { return nil, fmt.Errorf("ioscan classifier: no model endpoint resolved") } client = &openAIChatClient{ - endpoint: endpoint, - apiKey: c.APIKey, - model: model, - http: &http.Client{Timeout: classifierTimeout}, + endpoint: endpoint, + apiKey: c.APIKey, + model: model, + http: &http.Client{Timeout: classifierTimeout}, + retryDelay: classifierRetryDelay, } } return &LLMClassifier{client: client}, nil @@ -287,8 +290,24 @@ type openAIChatClient struct { apiKey string model string http *http.Client + // retryDelay is the pause before the single retry of a transient + // failure (see transientClassifierErr). + retryDelay time.Duration } +// transientClassifierErr marks a failure the endpoint is likely to recover +// from on the next request: a rate limit, a gateway/upstream 5xx, or a +// connection dropped before a reply. Timeouts are deliberately not transient +// so a hung backend costs one classifierTimeout per string, not two. +type transientClassifierErr struct{ err error } + +func (e *transientClassifierErr) Error() string { return e.err.Error() } +func (e *transientClassifierErr) Unwrap() error { return e.err } + +// Complete sends one chat completion. A transient failure is retried once +// after retryDelay so a single gateway blip does not make the scheduler pass +// untrusted text on unclassified (fail-open) or withhold it (fail-closed) +// (hivecommons/hive#18105). func (c *openAIChatClient) Complete(ctx context.Context, messages []chatMessage, maxTokens int) (string, error) { body, err := json.Marshal(chatRequest{ Model: c.model, @@ -299,6 +318,48 @@ func (c *openAIChatClient) Complete(ctx context.Context, messages []chatMessage, if err != nil { return "", err } + reply, err := c.completeOnce(ctx, body) + var transient *transientClassifierErr + if err == nil || !errors.As(err, &transient) { + return reply, err + } + if c.retryDelay > 0 { + t := time.NewTimer(c.retryDelay) + select { + case <-ctx.Done(): + t.Stop() + return "", err + case <-t.C: + } + } + reply, retryErr := c.completeOnce(ctx, body) + if retryErr != nil { + return "", fmt.Errorf("%w (after 1 retry)", retryErr) + } + return reply, nil +} + +// retryableStatus reports whether an HTTP status from the classifier model +// endpoint is worth one retry. +func retryableStatus(code int) bool { + switch code { + case http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, + http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + } + return false +} + +// isTimeoutErr reports whether err is a request or dial timeout. +func isTimeoutErr(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var te interface{ Timeout() bool } + return errors.As(err, &te) && te.Timeout() +} + +func (c *openAIChatClient) completeOnce(ctx context.Context, body []byte) (string, error) { reqCtx, cancel := context.WithTimeout(ctx, classifierTimeout) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, c.endpoint+"/v1/chat/completions", bytes.NewReader(body)) @@ -311,11 +372,18 @@ func (c *openAIChatClient) Complete(ctx context.Context, messages []chatMessage, } resp, err := c.http.Do(req) if err != nil { - return "", err + if ctx.Err() != nil || isTimeoutErr(err) { + return "", err + } + return "", &transientClassifierErr{err: err} } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("classifier model returned %d", resp.StatusCode) + statusErr := fmt.Errorf("classifier model returned %d", resp.StatusCode) + if retryableStatus(resp.StatusCode) { + return "", &transientClassifierErr{err: statusErr} + } + return "", statusErr } respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { diff --git a/src/pkg/ioscan/classifier_retry_test.go b/src/pkg/ioscan/classifier_retry_test.go new file mode 100644 index 0000000000..3c89ee9f0f --- /dev/null +++ b/src/pkg/ioscan/classifier_retry_test.go @@ -0,0 +1,208 @@ +package ioscan + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +const benignReply = `{"choices":[{"message":{"content":"{\"score\":0.1,\"category\":\"benign\",\"rationale\":\"ok\"}"}}]}` + +// A single gateway blip must not reach the scheduler's fail mode: the +// transient statuses are retried once and the second reply is used +// (hivecommons/hive#18105). +func TestOpenAIChatClientRetriesTransientStatusOnce(t *testing.T) { + for _, status := range []int{ + http.StatusTooManyRequests, + http.StatusInternalServerError, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, + } { + t.Run(fmt.Sprint(status), func(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(status) + return + } + fmt.Fprint(w, benignReply) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + reply, err := client.Complete(context.Background(), nil, 10) + if err != nil { + t.Fatalf("Complete: %v, want the retry to succeed", err) + } + if !strings.Contains(reply, `"category":"benign"`) { + t.Fatalf("reply = %q, want the retried reply", reply) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2 (one failure, one retry)", got) + } + }) + } +} + +// A persistent transient failure is retried exactly once, then surfaces with +// the original status so the fail-mode WARN still names it. +func TestOpenAIChatClientGivesUpAfterOneRetry(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + _, err := client.Complete(context.Background(), nil, 10) + if err == nil || !strings.Contains(err.Error(), "returned 503") || !strings.Contains(err.Error(), "after 1 retry") { + t.Fatalf("Complete error = %v, want the 503 noted as retried", err) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2", got) + } +} + +// Client errors (bad key, unknown model) will not fix themselves: no retry. +func TestOpenAIChatClientDoesNotRetryPermanentStatus(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusNotFound} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(status) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + _, err := client.Complete(context.Background(), nil, 10) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("returned %d", status)) { + t.Fatalf("Complete error = %v, want the %d status", err, status) + } + if strings.Contains(err.Error(), "retry") { + t.Fatalf("Complete error = %v, want no retry for a permanent status", err) + } + if got := calls.Load(); got != 1 { + t.Fatalf("calls = %d, want 1", got) + } + }) + } +} + +// A connection dropped before any reply (gateway restart) is retried. +func TestOpenAIChatClientRetriesDroppedConnection(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("response writer cannot hijack") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + _ = conn.Close() + return + } + fmt.Fprint(w, benignReply) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + if _, err := client.Complete(context.Background(), nil, 10); err != nil { + t.Fatalf("Complete: %v, want the retry after a dropped connection to succeed", err) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2", got) + } +} + +// A timeout is not retried: a hung backend must cost one timeout per string, +// not two. +func TestOpenAIChatClientDoesNotRetryTimeout(t *testing.T) { + var calls atomic.Int32 + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + select { + case <-release: + case <-r.Context().Done(): + } + })) + defer srv.Close() + defer close(release) + httpClient := srv.Client() + httpClient.Timeout = 50 * time.Millisecond + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: httpClient} + if _, err := client.Complete(context.Background(), nil, 10); err == nil { + t.Fatal("Complete: expected a timeout error") + } + if got := calls.Load(); got != 1 { + t.Fatalf("calls = %d, want 1 (timeouts are not retried)", got) + } +} + +// cancelAfterTransport cancels the caller's context as soon as the first +// response arrives, so the cancellation lands in the retry pause. +type cancelAfterTransport struct { + next http.RoundTripper + cancel context.CancelFunc +} + +func (c cancelAfterTransport) RoundTrip(r *http.Request) (*http.Response, error) { + resp, err := c.next.RoundTrip(r) + c.cancel() + return resp, err +} + +// A caller that cancels during the retry pause gets the original error back +// without a second request. +func TestOpenAIChatClientRetryHonoursCancellation(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusBadGateway) + })) + defer srv.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + httpClient := srv.Client() + httpClient.Transport = cancelAfterTransport{next: httpClient.Transport, cancel: cancel} + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: httpClient, retryDelay: time.Hour} + done := make(chan error, 1) + go func() { + _, err := client.Complete(ctx, nil, 10) + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "returned 502") { + t.Fatalf("Complete error = %v, want the original 502", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Complete did not return after cancellation during the retry pause") + } + if got := calls.Load(); got != 1 { + t.Fatalf("calls = %d, want 1 (no retry after cancellation)", got) + } +} + +func TestNewLLMClassifierSetsRetryDelay(t *testing.T) { + c, err := NewLLMClassifier(LLMClassifierConfig{Endpoint: "https://models.example"}) + if err != nil { + t.Fatalf("NewLLMClassifier: %v", err) + } + client, ok := c.client.(*openAIChatClient) + if !ok { + t.Fatalf("client = %T, want the built-in OpenAI-compatible client", c.client) + } + if client.retryDelay != classifierRetryDelay { + t.Fatalf("retryDelay = %v, want %v", client.retryDelay, classifierRetryDelay) + } +} From 3349b8551e65ee2a2ad8f984c3f4710e9203150c Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:19:42 -0500 Subject: [PATCH 156/223] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20warn?= =?UTF-8?q?=20once=20per=20author=20and=20repo=20for=20unsearchable=20cont?= =?UTF-8?q?ributor=20search=20(#18089)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(automerge): warn once per author and repo for unsearchable contributor search The contributor-lane 422 warning was keyed per (author, repository) pair, so one repository the App cannot see warned once for every contributor, and one unsearchable author warned once for every hive repository. TunaOS logged it 15 times in 10m and self-reported it as unclassified_error_repeated. Refusals are still remembered per pair for the TTL, but WARN now fires only when neither the author nor the searched repository has warned before; everything else logs at Debug. Fixes #18059 Hive-Run: hivecommons/hive#18059 Hive-Plan: selfreport-unclassified-error-repeated Hive-Spec: contributor-lane-auto-promotion#unsearchable-warn-once Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🧪 automerge: pin the 4096-entry cap on the unsearchable maps The mutation run on #18089 left both `len(...) > 4096` resets in noteContributorUnsearchable surviving as `>=`. Add a boundary test: a map holding exactly 4096 entries is kept, and one past the cap is reset. Each `>=` mutant now fails the test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...xed-18059-contributor-unsearchable-warn.md | 1 + .../automerge_contributor_unsearchable.go | 29 ++++++-- ...automerge_contributor_unsearchable_test.go | 67 +++++++++++++++++++ src/pkg/github/automerge/automerge_sweep.go | 8 ++- 4 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-18059-contributor-unsearchable-warn.md diff --git a/changelog.d/fixed-18059-contributor-unsearchable-warn.md b/changelog.d/fixed-18059-contributor-unsearchable-warn.md new file mode 100644 index 0000000000..75dd2ae587 --- /dev/null +++ b/changelog.d/fixed-18059-contributor-unsearchable-warn.md @@ -0,0 +1 @@ +- Contributor-lane auto-promotion now warns once per refused author and once per refused repository when GitHub's search cannot see a repository, instead of once per author × repository pair, so one invisible repository or one unsearchable author no longer floods WARN and self-reports as a repeated unclassified error (#18059). diff --git a/src/pkg/github/automerge/automerge_contributor_unsearchable.go b/src/pkg/github/automerge/automerge_contributor_unsearchable.go index ee9714616b..eab10b38e9 100644 --- a/src/pkg/github/automerge/automerge_contributor_unsearchable.go +++ b/src/pkg/github/automerge/automerge_contributor_unsearchable.go @@ -17,8 +17,12 @@ import ( // TTL lets a repaired repository or account come back without a restart. const contributorUnsearchableTTL = time.Hour -// contributorUnsearchableMessage is logged once per (author, repository) -// pair per process at WARN, then at Debug. +// contributorUnsearchableMessage is logged at WARN only for a refusal whose +// author and searched repository have both not been warned for yet in this +// process, then at Debug. Keying the WARN by pair let one invisible +// repository warn once for every contributor (or one unsearchable author once +// for every hive repository), which the self-report classifier counted as a +// repeated unclassified error (hivecommons/hive#18059). const contributorUnsearchableMessage = "contributor-lane auto-promotion search cannot see repository for this author; leaving it out of the hive-wide count" // searchQueryUnsearchable reports whether err is GitHub's 422 Validation @@ -45,20 +49,31 @@ func (c *Engine) contributorSearchSkips(author string, ref contributorRepoRef) b } // noteContributorUnsearchable records that GitHub refused to search ref for -// author, so the next contributorUnsearchableTTL of sweeps leave it out. The -// first refusal for a pair logs at WARN; a refusal again after the TTL logs -// at Debug, so a permanently unsearchable pair is not a repeated warning. +// author, so the next contributorUnsearchableTTL of sweeps leave it out. A +// refusal logs at WARN only when neither its author nor its searched +// repository has warned before in this process; every other refusal — +// another author for the same invisible repository, another repository for +// the same unsearchable author, or the same pair again after the TTL — logs +// at Debug, so the warnings stay bounded by the distinct causes rather than +// growing with authors × repositories. func (c *Engine) noteContributorUnsearchable(owner, repo, author string, ref contributorRepoRef, err error) { key := contributorUnsearchableKey(author, ref) + authorKey := "author:" + strings.ToLower(author) + repoKey := "repo:" + strings.ToLower(ref.Owner+"/"+ref.Repo) c.contributorUnsearchableMu.Lock() if c.contributorUnsearchable == nil || len(c.contributorUnsearchable) > 4096 { c.contributorUnsearchable = map[string]time.Time{} } - _, seen := c.contributorUnsearchable[key] + if c.contributorUnsearchableWarned == nil || len(c.contributorUnsearchableWarned) > 4096 { + c.contributorUnsearchableWarned = map[string]bool{} + } c.contributorUnsearchable[key] = c.clock().Add(contributorUnsearchableTTL) + warned := c.contributorUnsearchableWarned[authorKey] || c.contributorUnsearchableWarned[repoKey] + c.contributorUnsearchableWarned[authorKey] = true + c.contributorUnsearchableWarned[repoKey] = true c.contributorUnsearchableMu.Unlock() args := []any{"repo", owner + "/" + repo, "author", author, "searched_repo", ref.Owner + "/" + ref.Repo, "retry_after", contributorUnsearchableTTL, "error", err} - if seen { + if warned { c.debug(contributorUnsearchableMessage, args...) return } diff --git a/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go b/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go index 99d84bb503..c30cd3ee89 100644 --- a/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go +++ b/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go @@ -3,6 +3,8 @@ package automerge import ( "bytes" "context" + "errors" + "fmt" "log/slog" "strings" "testing" @@ -72,3 +74,68 @@ func TestContributorMergedReposRemembersUnsearchableRepository(t *testing.T) { t.Fatalf("warning should name the refused repository:\n%s", out) } } + +// Regression for hivecommons/hive#18059: one author refused for several +// repositories, and several authors refused for the same repository, warn +// once in total instead of once per (author, repository) pair — the pair +// product flooded WARN (15 in 10m on one hive) and self-reported as an +// unclassified repeated error. Every refusal is still remembered per pair. +func TestContributorUnsearchableWarnsOncePerAuthorAndRepository(t *testing.T) { + fx := validContributorFixture() + fx.unsearchable = map[string]bool{"acme/docs": true, "acme/homebrew-hive": true} + api := newContributorLaneAPI(t, &fx) + defer api.Close() + var logs bytes.Buffer + clock := &contributorLaneClock{at: time.Date(2026, 11, 13, 3, 0, 0, 0, time.UTC)} + engine := newContributorLaneEngineRepos(api.URL, []string{"widget", "hive", "docs", "homebrew-hive"}, defaultContributorLanePolicy(), clock, Options{ + Logger: slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + authors := []string{contributorLaneTestAuthor, "thatguywob", "another-dev"} + for _, author := range authors { + engine.contributorMergedRepos(context.Background(), "acme", "widget", author) + } + + out := logs.String() + if n := strings.Count(out, contributorUnsearchableMessage); n != 1 { + t.Fatalf("unsearchable warning logged %d times for %d authors × 2 refused repositories, want 1:\n%s", n, len(authors), out) + } + for _, author := range authors { + for _, refused := range []string{"docs", "homebrew-hive"} { + if !engine.contributorSearchSkips(author, contributorRepoRef{Owner: "acme", Repo: refused}) { + t.Fatalf("refusal of acme/%s for %s was not remembered", refused, author) + } + } + } +} + +// The pair TTL map and the WARN-dedup map are each bounded at 4096 entries: +// a map holding exactly 4096 is kept, and only one past the cap is reset, so +// the cap is the documented size and not one entry fewer. +func TestNoteContributorUnsearchableResetsOnlyPastCap(t *testing.T) { + const capacity = 4096 + engine := &Engine{ + contributorUnsearchable: map[string]time.Time{}, + contributorUnsearchableWarned: map[string]bool{}, + } + for i := 0; i < capacity; i++ { + engine.contributorUnsearchable[fmt.Sprintf("filler-%d", i)] = time.Time{} + engine.contributorUnsearchableWarned[fmt.Sprintf("filler-%d", i)] = true + } + refusal := errors.New("422") + + engine.noteContributorUnsearchable("acme", "widget", "first-dev", contributorRepoRef{Owner: "acme", Repo: "docs"}, refusal) + if got := len(engine.contributorUnsearchable); got != capacity+1 { + t.Fatalf("pair map at the cap: len = %d, want %d (kept, new pair added)", got, capacity+1) + } + if got := len(engine.contributorUnsearchableWarned); got != capacity+2 { + t.Fatalf("warned map at the cap: len = %d, want %d (kept, author and repo added)", got, capacity+2) + } + + engine.noteContributorUnsearchable("acme", "widget", "second-dev", contributorRepoRef{Owner: "acme", Repo: "wiki"}, refusal) + if got := len(engine.contributorUnsearchable); got != 1 { + t.Fatalf("pair map past the cap: len = %d, want 1 (reset, new pair added)", got) + } + if got := len(engine.contributorUnsearchableWarned); got != 2 { + t.Fatalf("warned map past the cap: len = %d, want 2 (reset, author and repo added)", got) + } +} diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index e0afef2944..0f8d9375fe 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -297,8 +297,12 @@ type Engine struct { // contributorUnsearchable maps author|owner/repo to the time until which // contributorMergedRepos leaves that pair out of its search, after // GitHub refused to search it (see automerge_contributor_unsearchable.go). - contributorUnsearchableMu sync.Mutex - contributorUnsearchable map[string]time.Time + // contributorUnsearchableWarned holds the "author:<login>" and + // "repo:<owner/repo>" keys a refusal has already warned for, so the + // author × repository product logs WARN at most once per side. + contributorUnsearchableMu sync.Mutex + contributorUnsearchable map[string]time.Time + contributorUnsearchableWarned map[string]bool mergerAuthzMu sync.RWMutex // humanReturnChecked remembers, per PR key, the head whose worn From def19fdde8e560275dd67e4d7a770e617484b97d Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 18:21:28 +0000 Subject: [PATCH 157/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.244.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.244.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-benchmarks-links.md | 1 - changelog.d/fixed-18092-knowledge-check-toast.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-benchmarks-links.md delete mode 100644 changelog.d/fixed-18092-knowledge-check-toast.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f6020238..b69a348f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.244.0) + +### Added + +- Hub header nav and My Hives now link to the fleet benchmarks page (hivecommons.dev/benchmarks), and the spoke dashboard FAQ points at it for model comparisons (#18034). + +### Fixed + +- Knowledge Integrations: the per-connector **Check** button now reports its result — a success/info toast with the sign-in state (and the card refreshed), or an error toast carrying the server's `error` string verbatim (for example `no knowledge connectors are configured`) instead of a "checking" toast followed by silence ([#18092](https://github.com/hivecommons/hive/issues/18092)). + ## 2026-11-13 (v5.243.0) ### Added diff --git a/changelog.d/added-benchmarks-links.md b/changelog.d/added-benchmarks-links.md deleted file mode 100644 index 26cf95b2bb..0000000000 --- a/changelog.d/added-benchmarks-links.md +++ /dev/null @@ -1 +0,0 @@ -- Hub header nav and My Hives now link to the fleet benchmarks page (hivecommons.dev/benchmarks), and the spoke dashboard FAQ points at it for model comparisons (#18034). diff --git a/changelog.d/fixed-18092-knowledge-check-toast.md b/changelog.d/fixed-18092-knowledge-check-toast.md deleted file mode 100644 index 1500f367c0..0000000000 --- a/changelog.d/fixed-18092-knowledge-check-toast.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge Integrations: the per-connector **Check** button now reports its result — a success/info toast with the sign-in state (and the card refreshed), or an error toast carrying the server's `error` string verbatim (for example `no knowledge connectors are configured`) instead of a "checking" toast followed by silence ([#18092](https://github.com/hivecommons/hive/issues/18092)). From d084388023409461c49a62f55937bc3a4777e2b5 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:38:13 -0500 Subject: [PATCH 158/223] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20make=20the?= =?UTF-8?q?=20narrow=20navbar=20agents=20pill=20a=20real,=20uncapped=20but?= =?UTF-8?q?ton=20(#18114)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The narrow top-bar "3 agents ▾" pill was a CSS ::after pseudo-element: its text counted the (at most 3) visible tiles rather than the agents in the run order, and a pseudo-element can never be a click target, so it did nothing when clicked. Replace it with a <button data-action="openAgentsUpNextPanel"> built in renderAgentNavbarUpNext from textContent, kept as the strip's last child so the tile diff's index-based insertBefore still orders tiles ahead of it. The label is the uncapped count of scheduled agents ("9 agents", "1 agent"), no caret, with a title/aria-label naming the action. The 340px container query shows the button; the 220px query swaps in a "Next" span instead of generated content. Hive-Run: hivecommons/hive#18110 Hive-Plan: issue-18110 "How to build it" Hive-Spec: hivecommons/hive#18110#acceptance-criteria Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...ixed-18110-navbar-agents-summary-button.md | 1 + .../dashboard/agent_collapsed_tiles_test.go | 7 +- .../dashboard/agent_navbar_summary_ui_test.go | 140 ++++++++++++++++++ src/pkg/dashboard/navbar_upgrade_bee_test.go | 4 +- .../static-src/css/sections-collapse.css | 10 +- src/pkg/dashboard/static-src/js/sections.js | 42 +++++- src/pkg/dashboard/static/index.html | 52 ++++++- 7 files changed, 241 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-18110-navbar-agents-summary-button.md create mode 100644 src/pkg/dashboard/agent_navbar_summary_ui_test.go diff --git a/changelog.d/fixed-18110-navbar-agents-summary-button.md b/changelog.d/fixed-18110-navbar-agents-summary-button.md new file mode 100644 index 0000000000..7c48d4aa50 --- /dev/null +++ b/changelog.d/fixed-18110-navbar-agents-summary-button.md @@ -0,0 +1 @@ +- Dashboard top bar: when the centre column is narrow, the agent run-order strip's summary pill is now a real button that opens the Agents section (click, Enter or Space), and it shows how many scheduled agents are in the run order ("9 agents", "1 agent") instead of a count capped at 3; the misleading "▾" caret is gone ([#18110](https://github.com/hivecommons/hive/issues/18110)). diff --git a/src/pkg/dashboard/agent_collapsed_tiles_test.go b/src/pkg/dashboard/agent_collapsed_tiles_test.go index 1895afb12a..c4acfd6db0 100644 --- a/src/pkg/dashboard/agent_collapsed_tiles_test.go +++ b/src/pkg/dashboard/agent_collapsed_tiles_test.go @@ -46,7 +46,8 @@ func TestAgentsSidebarLinkAndNavbarUpNextContracts(t *testing.T) { `id="agent-navbar-upnext"`, `class="agent-navbar-upnext"`, `function renderAgentNavbarUpNext(agents, nowMs)`, - `agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)).slice(0, 3)`, + `const scheduled = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now));`, + `const list = scheduled.slice(0, 3);`, `function renderAgentNavbarTileDiff(wrap, list, now)`, `data-agent-key`, `function openAgentsUpNextPanel()`, @@ -54,8 +55,8 @@ func TestAgentsSidebarLinkAndNavbarUpNextContracts(t *testing.T) { `.agent-navbar-tile { flex: 0 0 auto; min-width: 136px;`, `.agent-navbar-tile .agent-tile-name { grid-column: 1; grid-row: 1; min-width: max-content; overflow: visible; text-overflow: clip; }`, `@container navbar-center (max-width: 560px)`, - `.agent-navbar-upnext::after { content: attr(data-agent-summary);`, - `wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : '';`, + `.agent-navbar-summary { flex: 0 0 auto; display: none;`, + `const summary = renderAgentNavbarSummary(wrap, scheduled.length);`, `renderAgentNavbarUpNext(agents, Date.now())`, `refreshAgentNavbarUpNextCountdowns(now);`, `@media (max-width: 900px) { .agent-navbar-upnext { display: none !important; } }`, diff --git a/src/pkg/dashboard/agent_navbar_summary_ui_test.go b/src/pkg/dashboard/agent_navbar_summary_ui_test.go new file mode 100644 index 0000000000..3c3c2df114 --- /dev/null +++ b/src/pkg/dashboard/agent_navbar_summary_ui_test.go @@ -0,0 +1,140 @@ +package dashboard + +import ( + "os/exec" + "strings" + "testing" +) + +// When the navbar centre column is narrow the up-next strip collapses to a +// single summary control. It used to be a CSS ::after pseudo-element whose +// text was capped at 3 (the tile slice) and which could not receive clicks +// (#18110). These tests execute the real render functions against a tiny fake +// DOM so the count, the action wiring and the "no button when empty" rule are +// exercised, not just pattern-matched. +func TestAgentNavbarSummaryButtonCountAndAction(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node unavailable: agent navbar summary behavior was not executed") + } + html := indexHTML(t) + block := jsFunc(t, html, "agentNavbarSummaryLabel") + "\n" + + jsFunc(t, html, "renderAgentNavbarSummary") + "\n" + + jsFunc(t, html, "renderAgentNavbarUpNext") + + script := ` +class El { + constructor(tag) { this.tagName = tag.toUpperCase(); this.children = []; this.attrs = {}; this.className = ''; this._text = ''; this.parentNode = null; this.hidden = false; this.dataset = {}; } + setAttribute(k, v) { if (k === 'class') this.className = String(v); else this.attrs[k] = String(v); } + getAttribute(k) { if (k === 'class') return this.className; return Object.prototype.hasOwnProperty.call(this.attrs, k) ? this.attrs[k] : null; } + appendChild(c) { if (c.parentNode) c.parentNode.removeChild(c); c.parentNode = this; this.children.push(c); return c; } + insertBefore(c, ref) { if (!ref) return this.appendChild(c); if (c.parentNode) c.parentNode.removeChild(c); const i = this.children.indexOf(ref); c.parentNode = this; this.children.splice(i, 0, c); return c; } + removeChild(c) { this.children = this.children.filter(x => x !== c); c.parentNode = null; return c; } + remove() { if (this.parentNode) this.parentNode.removeChild(this); } + get lastElementChild() { return this.children[this.children.length - 1] || null; } + get textContent() { return this.children.length ? this.children.map(c => c.textContent).join('') : this._text; } + set textContent(v) { this.children = []; this._text = String(v); } + querySelector(sel) { const cls = sel.replace(/^\./, ''); for (const c of this.children) { if (c.className.split(/\s+/).includes(cls)) return c; const d = c.querySelector(sel); if (d) return d; } return null; } + querySelectorAll(sel) { const cls = sel.replace(/^\./, ''); const out = []; const walk = el => el.children.forEach(c => { if (c.className.split(/\s+/).includes(cls)) out.push(c); walk(c); }); walk(this); return out; } +} +const wrap = new El('span'); +wrap.setAttribute('id', 'agent-navbar-upnext'); +const document = { getElementById: id => id === 'agent-navbar-upnext' ? wrap : null, createElement: tag => new El(tag) }; +const window = { _lastStatus: { governor: {} } }; +function agentTileStates(agents) { return agents.map(a => ({ id: a.name, agent: a })); } +function agentNavbarHasSchedule(a) { return a.scheduled !== false; } +// Mirror the real diff's index-based insertBefore so tile placement relative +// to the summary button is exercised. +function renderAgentNavbarTileDiff(w, list) { + w.querySelectorAll('.agent-navbar-tile').forEach(t => t.remove()); + list.forEach((item, i) => { const tile = new El('button'); tile.className = 'agent-navbar-tile'; tile.setAttribute('data-agent-key', item.id); w.insertBefore(tile, w.children[i] || null); }); +} +function ensureAgentsCollapsedCountdownTicker() {} +` + block + ` +function agents(n, extraUnscheduled) { + const out = []; + for (let i = 0; i < n; i++) out.push({ name: 'a' + i }); + for (let i = 0; i < (extraUnscheduled || 0); i++) out.push({ name: 'off' + i, scheduled: false }); + return out; +} +function fail(msg) { throw new Error(msg); } +function summary() { return wrap.querySelector('.agent-navbar-summary'); } +function check(name, n, extra, wantText) { + renderAgentNavbarUpNext(agents(n, extra), 0); + const btn = summary(); + if (!btn) fail(name + ': summary button missing'); + if (btn.tagName !== 'BUTTON' || btn.getAttribute('type') !== 'button') fail(name + ': summary is not a type=button <button>'); + if (btn.getAttribute('data-action') !== 'openAgentsUpNextPanel') fail(name + ': data-action = ' + btn.getAttribute('data-action')); + const full = btn.querySelector('.agent-navbar-summary-full'); + if (!full || full.textContent !== wantText) fail(name + ': text = ' + (full && full.textContent) + ', want ' + wantText); + if (!btn.querySelector('.agent-navbar-summary-short')) fail(name + ': narrow "Next" span missing'); + if (btn.textContent.includes('▾')) fail(name + ': summary still shows a dropdown caret'); + const title = btn.getAttribute('title') || ''; + if (!title.startsWith(wantText + ' in the run order') || btn.getAttribute('aria-label') !== title) fail(name + ': title/aria-label = ' + title); + if (wrap.hidden) fail(name + ': strip hidden with scheduled agents'); + if (wrap.lastElementChild !== btn) fail(name + ': summary is not the last child of the strip'); + const tiles = wrap.querySelectorAll('.agent-navbar-tile'); + if (tiles.length !== Math.min(n, 3)) fail(name + ': tiles = ' + tiles.length); + if (wrap.querySelectorAll('.agent-navbar-summary').length !== 1) fail(name + ': duplicate summary buttons'); +} +check('nine scheduled agents are counted past the 3-tile cap', 9, 2, '9 agents'); +check('a single scheduled agent reads singular', 1, 0, '1 agent'); +check('two scheduled agents read plural', 2, 3, '2 agents'); +renderAgentNavbarUpNext(agents(0, 4), 0); +if (summary()) fail('no scheduled agents must not render a summary button'); +if (!wrap.hidden) fail('no scheduled agents must hide the strip'); +check('the button comes back after an empty render', 4, 0, '4 agents'); +` + cmd := exec.Command(node, "-e", script) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("agent navbar summary JS failed: %v\n%s", err, out) + } +} + +func TestAgentNavbarSummaryStaticCSS(t *testing.T) { + html := indexHTML(t) + for _, gone := range []string{ + `content: attr(data-agent-summary)`, + `.agent-navbar-upnext::after`, + `agents ▾`, + } { + if strings.Contains(html, gone) { + t.Fatalf("static/index.html still contains the unclickable pseudo-element pill %q", gone) + } + } + base := cssRule(t, html, ".agent-navbar-summary") + for _, want := range []string{"display: none", "cursor: pointer", "min-height: calc(var(--nav-chip-h) + var(--sp-4))"} { + if !strings.Contains(base, want) { + t.Fatalf(".agent-navbar-summary rule missing %q: %s", want, base) + } + } + if !strings.Contains(html, ".agent-navbar-summary:hover, .agent-navbar-summary:focus-visible {") { + t.Fatal(".agent-navbar-summary has no :focus-visible style") + } + containerBlock := func(query string) string { + i := strings.Index(html, "@container navbar-center ("+query+") {") + if i < 0 { + t.Fatalf("static/index.html missing @container navbar-center (%s)", query) + } + end := strings.Index(html[i:], "\n }") + if end < 0 { + t.Fatalf("unterminated @container navbar-center (%s)", query) + } + return html[i : i+end] + } + if q := containerBlock("max-width: 340px"); !strings.Contains(q, ".agent-navbar-upnext .agent-navbar-summary { display: inline-flex; }") { + t.Fatalf("340px container query does not show the summary button:\n%s", q) + } + q := containerBlock("max-width: 220px") + for _, want := range []string{ + ".agent-navbar-summary .agent-navbar-summary-full { display: none; }", + ".agent-navbar-summary .agent-navbar-summary-short { display: inline; }", + } { + if !strings.Contains(q, want) { + t.Fatalf("220px container query missing %q:\n%s", want, q) + } + } + if strings.Contains(q, "content:") { + t.Fatalf("220px container query must toggle a span, not generate content:\n%s", q) + } +} diff --git a/src/pkg/dashboard/navbar_upgrade_bee_test.go b/src/pkg/dashboard/navbar_upgrade_bee_test.go index 835c09ecf7..dc686033cd 100644 --- a/src/pkg/dashboard/navbar_upgrade_bee_test.go +++ b/src/pkg/dashboard/navbar_upgrade_bee_test.go @@ -101,13 +101,13 @@ func TestNavbarUpgradePillCannotCoverCenterAgents(t *testing.T) { `.oc-topbar-right { position: relative; z-index: 2;`, `flex-wrap: nowrap;`, `.oc-navbar-upgrade { --oc-version-bee-lane: 56px; flex: 0 1 min(260px, 24vw); min-width: 0; max-width: min(260px, 24vw);`, - `.agent-navbar-upnext::after { content: attr(data-agent-summary);`, + `.agent-navbar-summary { flex: 0 0 auto; display: none;`, `@container navbar-center (max-width: 560px)`, `.agent-navbar-tile .agent-tile-next { display: none; }`, `@container navbar-center (max-width: 340px)`, `.agent-navbar-upnext .agent-navbar-tile { display: none; }`, `@media (max-width: 900px) { .agent-navbar-upnext { display: none !important; } }`, - `wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : '';`, + `const summary = renderAgentNavbarSummary(wrap, scheduled.length);`, `const detail = versionUpgradeProgressStatus(v || {}, progress);`, } { if !strings.Contains(html, want) { diff --git a/src/pkg/dashboard/static-src/css/sections-collapse.css b/src/pkg/dashboard/static-src/css/sections-collapse.css index 793af8e0d4..cb73116104 100644 --- a/src/pkg/dashboard/static-src/css/sections-collapse.css +++ b/src/pkg/dashboard/static-src/css/sections-collapse.css @@ -127,7 +127,9 @@ .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } - .agent-navbar-upnext::after { content: attr(data-agent-summary); display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary { flex: 0 0 auto; display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); cursor: pointer; font: inherit; font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary:hover, .agent-navbar-summary:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } + .agent-navbar-summary-short { display: none; } @container navbar-center (max-width: 560px) { .agent-navbar-tile { min-width: 104px; } .agent-navbar-tile .agent-tile-next { display: none; } @@ -140,10 +142,12 @@ @container navbar-center (max-width: 340px) { .agent-navbar-upnext { overflow: hidden; mask-image: none; } .agent-navbar-upnext .agent-navbar-tile { display: none; } - .agent-navbar-upnext::after { display: inline-flex; } + .agent-navbar-upnext .agent-navbar-summary { display: inline-flex; } } @container navbar-center (max-width: 220px) { - .agent-navbar-upnext::after { content: 'Next'; padding-inline: var(--sp-1); } + .agent-navbar-upnext .agent-navbar-summary { padding-inline: var(--sp-1); } + .agent-navbar-summary .agent-navbar-summary-full { display: none; } + .agent-navbar-summary .agent-navbar-summary-short { display: inline; } } @keyframes navbar-up-next-pulse { 0% { box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--amber) 72%, transparent), 0 0 0 var(--sp-0) color-mix(in srgb, var(--amber) 0%, transparent); } diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index 35e32dd311..85b6a94e2b 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -1163,14 +1163,52 @@ next kick: ${agentCollapsedTimestampDetail(nextMs, a && (a.nextKickAt || a.nextK wrap.dataset.upNextKey = nextItem ? nextItem.id : ''; animateAgentNavbarReorder(wrap, beforeRects); } + function agentNavbarSummaryLabel(count) { + return String(count) + (count === 1 ? ' agent' : ' agents'); + } + // The narrow-strip summary is a real button (not a CSS pseudo-element) so + // it can carry data-action and receive clicks and keyboard activation. It + // stays the strip's LAST child so renderAgentNavbarTileDiff's index-based + // insertBefore always places tiles ahead of it. + function renderAgentNavbarSummary(wrap, count) { + let btn = wrap.querySelector('.agent-navbar-summary'); + if (!count) { + if (btn) btn.remove(); + return null; + } + if (!btn) { + btn = document.createElement('button'); + btn.setAttribute('type', 'button'); + btn.className = 'agent-navbar-summary'; + btn.setAttribute('data-action', 'openAgentsUpNextPanel'); + btn.setAttribute('data-stop', '1'); + const full = document.createElement('span'); + full.className = 'agent-navbar-summary-full'; + const short = document.createElement('span'); + short.className = 'agent-navbar-summary-short'; + short.textContent = 'Next'; + btn.appendChild(full); + btn.appendChild(short); + } + const label = agentNavbarSummaryLabel(count); + const title = label + ' in the run order — open the Agents section'; + const full = btn.querySelector('.agent-navbar-summary-full'); + if (full && full.textContent !== label) full.textContent = label; + btn.setAttribute('title', title); + btn.setAttribute('aria-label', title); + if (wrap.lastElementChild !== btn) wrap.appendChild(btn); + return btn; + } function renderAgentNavbarUpNext(agents, nowMs) { const wrap = document.getElementById('agent-navbar-upnext'); if (!wrap) return; const now = Number.isFinite(nowMs) ? nowMs : Date.now(); - const list = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)).slice(0, 3); + const scheduled = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)); + const list = scheduled.slice(0, 3); wrap.hidden = list.length === 0; - wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : ''; + const summary = renderAgentNavbarSummary(wrap, scheduled.length); renderAgentNavbarTileDiff(wrap, list, now); + if (summary && wrap.lastElementChild !== summary) wrap.appendChild(summary); if (list.length) ensureAgentsCollapsedCountdownTicker(); } function refreshAgentNavbarUpNextCountdowns(nowMs) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 6359a27634..1cccd9750b 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -6119,7 +6119,9 @@ .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } - .agent-navbar-upnext::after { content: attr(data-agent-summary); display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary { flex: 0 0 auto; display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); cursor: pointer; font: inherit; font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary:hover, .agent-navbar-summary:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } + .agent-navbar-summary-short { display: none; } @container navbar-center (max-width: 560px) { .agent-navbar-tile { min-width: 104px; } .agent-navbar-tile .agent-tile-next { display: none; } @@ -6132,10 +6134,12 @@ @container navbar-center (max-width: 340px) { .agent-navbar-upnext { overflow: hidden; mask-image: none; } .agent-navbar-upnext .agent-navbar-tile { display: none; } - .agent-navbar-upnext::after { display: inline-flex; } + .agent-navbar-upnext .agent-navbar-summary { display: inline-flex; } } @container navbar-center (max-width: 220px) { - .agent-navbar-upnext::after { content: 'Next'; padding-inline: var(--sp-1); } + .agent-navbar-upnext .agent-navbar-summary { padding-inline: var(--sp-1); } + .agent-navbar-summary .agent-navbar-summary-full { display: none; } + .agent-navbar-summary .agent-navbar-summary-short { display: inline; } } @keyframes navbar-up-next-pulse { 0% { box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--amber) 72%, transparent), 0 0 0 var(--sp-0) color-mix(in srgb, var(--amber) 0%, transparent); } @@ -11582,14 +11586,52 @@ <h3 class="faq-group-title">Where to get help</h3> wrap.dataset.upNextKey = nextItem ? nextItem.id : ''; animateAgentNavbarReorder(wrap, beforeRects); } + function agentNavbarSummaryLabel(count) { + return String(count) + (count === 1 ? ' agent' : ' agents'); + } + // The narrow-strip summary is a real button (not a CSS pseudo-element) so + // it can carry data-action and receive clicks and keyboard activation. It + // stays the strip's LAST child so renderAgentNavbarTileDiff's index-based + // insertBefore always places tiles ahead of it. + function renderAgentNavbarSummary(wrap, count) { + let btn = wrap.querySelector('.agent-navbar-summary'); + if (!count) { + if (btn) btn.remove(); + return null; + } + if (!btn) { + btn = document.createElement('button'); + btn.setAttribute('type', 'button'); + btn.className = 'agent-navbar-summary'; + btn.setAttribute('data-action', 'openAgentsUpNextPanel'); + btn.setAttribute('data-stop', '1'); + const full = document.createElement('span'); + full.className = 'agent-navbar-summary-full'; + const short = document.createElement('span'); + short.className = 'agent-navbar-summary-short'; + short.textContent = 'Next'; + btn.appendChild(full); + btn.appendChild(short); + } + const label = agentNavbarSummaryLabel(count); + const title = label + ' in the run order — open the Agents section'; + const full = btn.querySelector('.agent-navbar-summary-full'); + if (full && full.textContent !== label) full.textContent = label; + btn.setAttribute('title', title); + btn.setAttribute('aria-label', title); + if (wrap.lastElementChild !== btn) wrap.appendChild(btn); + return btn; + } function renderAgentNavbarUpNext(agents, nowMs) { const wrap = document.getElementById('agent-navbar-upnext'); if (!wrap) return; const now = Number.isFinite(nowMs) ? nowMs : Date.now(); - const list = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)).slice(0, 3); + const scheduled = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)); + const list = scheduled.slice(0, 3); wrap.hidden = list.length === 0; - wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : ''; + const summary = renderAgentNavbarSummary(wrap, scheduled.length); renderAgentNavbarTileDiff(wrap, list, now); + if (summary && wrap.lastElementChild !== summary) wrap.appendChild(summary); if (list.length) ensureAgentsCollapsedCountdownTicker(); } function refreshAgentNavbarUpNextCountdowns(nowMs) { From bb32b6675e843d8bfd69ac7716c925e6e7e8428b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:46:56 +0000 Subject: [PATCH 159/223] feat: ship spektacular in the hive image (#18104) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-17888-ship-spektacular.md | 1 + src/Dockerfile | 27 +++++++++++++++++++++ src/Dockerfile.contributor | 27 +++++++++++++++++++++ src/docs/cli-pins.md | 1 + 4 files changed, 56 insertions(+) create mode 100644 changelog.d/added-17888-ship-spektacular.md diff --git a/changelog.d/added-17888-ship-spektacular.md b/changelog.d/added-17888-ship-spektacular.md new file mode 100644 index 0000000000..db2fb2c358 --- /dev/null +++ b/changelog.d/added-17888-ship-spektacular.md @@ -0,0 +1 @@ +- The hive image now ships the `spektacular` CLI (pinned 0.31.0, checksum-verified, amd64 and arm64), so the Inbox → RFC destination runs `spektacular voice --publish` in-pod without an agent fallback (#17888). diff --git a/src/Dockerfile b/src/Dockerfile index e802b98fed..fd60ca9443 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -381,6 +381,33 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ echo "WARN: Goose CLI download failed — skipping (backend: goose will be unavailable)"; \ fi +# --- Layer 8.1: spektacular CLI (Inbox -> RFC destination) --- +# Pinned hivecommons/spektacular release (includes `spektacular voice`, +# hivecommons/spektacular#123), SHA-256-verified before extraction, installed +# to /usr/local/bin/spektacular so inbox.spek.binary / runs.spektacular.binary +# resolve on PATH with no config. Download failures are tolerated (the Inbox +# falls back to inbox.spek.agent); checksum mismatches hard-fail the build. +# Bump note: see src/docs/cli-pins.md (spektacular row); update the version and +# both digests together from the release's checksums.txt. +ARG SPEKTACULAR_VERSION=0.31.0 +ARG SPEKTACULAR_SHA256_AMD64=21bd077b6723de41185fa657c0e54ea5726c1c2928794548ada4a8776a7b4746 +ARG SPEKTACULAR_SHA256_ARM64=81d3b92c907bd6d66e4e1db79df24be8b64803dd26e0de1d7147f1ff5fc9b8d0 +RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$ARCH" in \ + amd64) SPEK_ARCH=x86_64; SPEK_SHA256="$SPEKTACULAR_SHA256_AMD64" ;; \ + arm64) SPEK_ARCH=arm64; SPEK_SHA256="$SPEKTACULAR_SHA256_ARM64" ;; \ + *) echo "unsupported arch for spektacular: $ARCH" >&2; exit 1 ;; \ + esac && \ + mkdir -p /tmp/spek-dl && \ + if curl -fsSL --retry 5 --retry-delay 5 --retry-connrefused "https://github.com/hivecommons/spektacular/releases/download/${SPEKTACULAR_VERSION}/spektacular_${SPEKTACULAR_VERSION}_linux_${SPEK_ARCH}.tar.gz" -o /tmp/spek-dl/spektacular.tar.gz; then \ + echo "${SPEK_SHA256} /tmp/spek-dl/spektacular.tar.gz" | sha256sum -c - && \ + tar xz -C /usr/local/bin -f /tmp/spek-dl/spektacular.tar.gz spektacular && \ + chmod +x /usr/local/bin/spektacular && \ + rm -rf /tmp/spek-dl; \ + else \ + echo "WARN: spektacular download failed - skipping (Inbox falls back to inbox.spek.agent)"; \ + fi + # --- Layer 8.25: Google Antigravity CLI (backend: agy) --- # The vendor distributes a self-contained Linux executable, not an npm package. # Pin the version, build ID, and SHA-512 from its public updater manifest; do diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index bc5e3349ad..3e6703fdf8 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -390,6 +390,33 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ echo "Goose install skipped"; \ fi +# spektacular CLI (Inbox -> RFC destination) --- +# Pinned hivecommons/spektacular release (includes `spektacular voice`, +# hivecommons/spektacular#123), SHA-256-verified before extraction, installed +# to /usr/local/bin/spektacular so inbox.spek.binary / runs.spektacular.binary +# resolve on PATH with no config. Download failures are tolerated (the Inbox +# falls back to inbox.spek.agent); checksum mismatches hard-fail the build. +# Bump note: see src/docs/cli-pins.md (spektacular row); update the version and +# both digests together from the release's checksums.txt. +ARG SPEKTACULAR_VERSION=0.31.0 +ARG SPEKTACULAR_SHA256_AMD64=21bd077b6723de41185fa657c0e54ea5726c1c2928794548ada4a8776a7b4746 +ARG SPEKTACULAR_SHA256_ARM64=81d3b92c907bd6d66e4e1db79df24be8b64803dd26e0de1d7147f1ff5fc9b8d0 +RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$ARCH" in \ + amd64) SPEK_ARCH=x86_64; SPEK_SHA256="$SPEKTACULAR_SHA256_AMD64" ;; \ + arm64) SPEK_ARCH=arm64; SPEK_SHA256="$SPEKTACULAR_SHA256_ARM64" ;; \ + *) echo "unsupported arch for spektacular: $ARCH" >&2; exit 1 ;; \ + esac && \ + mkdir -p /tmp/spek-dl && \ + if curl -fsSL --retry 5 --retry-delay 5 --retry-connrefused "https://github.com/hivecommons/spektacular/releases/download/${SPEKTACULAR_VERSION}/spektacular_${SPEKTACULAR_VERSION}_linux_${SPEK_ARCH}.tar.gz" -o /tmp/spek-dl/spektacular.tar.gz; then \ + echo "${SPEK_SHA256} /tmp/spek-dl/spektacular.tar.gz" | sha256sum -c - && \ + tar xz -C /usr/local/bin -f /tmp/spek-dl/spektacular.tar.gz spektacular && \ + chmod +x /usr/local/bin/spektacular && \ + rm -rf /tmp/spek-dl; \ + else \ + echo "WARN: spektacular download failed - skipping (Inbox falls back to inbox.spek.agent)"; \ + fi + # Install agy (Google Antigravity CLI). Not an npm package — Google ships it # as a standalone binary from a versioned, checksummed GCS URL (the same URL # Homebrew's antigravity-cli cask resolves to, confirmed against diff --git a/src/docs/cli-pins.md b/src/docs/cli-pins.md index e53751befa..4a430bed07 100644 --- a/src/docs/cli-pins.md +++ b/src/docs/cli-pins.md @@ -10,6 +10,7 @@ Every agent CLI hive drives is baked into the images at a fixed version: | pi | `PI_VERSION` (hub), `PI_CODING_AGENT_VERSION` (contributor) | npm `@earendil-works/pi-coding-agent` | npm | | Goose | `GOOSE_VERSION`, `GOOSE_SHA256_{AMD64,ARM64}` | GitHub releases `aaif-goose/goose` | SHA-256 per arch | | Antigravity (agy) | `AGY_VERSION` + `AGY_BUILD`, `AGY_SHA512_*` (hub); `AGY_VERSION=<ver>-<build>`, `AGY_SHA256_*` (contributor) | Google's updater artifact, as tracked by the Homebrew `antigravity-cli` cask | SHA-512 (hub) / SHA-256 (contributor) per arch | +| spektacular | `SPEKTACULAR_VERSION`, `SPEKTACULAR_SHA256_{AMD64,ARM64}` | GitHub releases `hivecommons/spektacular` | SHA-256 per arch, from the release `checksums.txt` (manual bump for now; not yet in `cli-pin-bump.sh`) | | Oh My Pi (omp) | `OMP_VERSION`, `OMP_SHA256_*` | GitHub releases `can1357/oh-my-pi` | SHA-256 per arch, cross-checked with `SHA256SUMS.txt` | | Muse Code | `MUSE_VERSION`, `MUSE_SHA256_*` | `api.meta.ai/muse-code/channels/muse-stable` and its release manifest | SHA-256 per arch, cross-checked with the manifest | | bobshell (bob) | `BOBSHELL_VERSION`, `BOBSHELL_SHA256_*` | `bobshell2-version.txt` on IBM COS | SHA-256, cross-checked with the vendor `.tgz.sha256` | From f63d1d235d10317cc95b220a8c1810b6838f90ce Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:47:47 -0500 Subject: [PATCH 160/223] =?UTF-8?q?=F0=9F=90=9B=20fleetreport:=20stop=20se?= =?UTF-8?q?lf-reporting=20completed=20Claude=20credential=20re-bridges=20(?= =?UTF-8?q?#18108)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code renames a fresh file over each agent's shared-login symlink on every token refresh, every agent refreshes within the same minute or two, and the watchdog re-bridges each link within 5s. That burst of successful repairs (already audited) crossed the unclassified_error_repeated threshold and filed a fleet self-report for designed, self-healed behaviour. Exclude exactly that completed-repair WARN from the slogfeed tee; it stays in local logs. Failed or refused repairs log other messages and remain reportable. Fixes #18107 Hive-Run: hivecommons/hive#18107 Hive-Plan: selfreport-triage Hive-Spec: fleetreport-slogfeed#expected-recovery-not-reported Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .../fixed-18107-claude-rebridge-selfreport.md | 1 + src/crux/agent/claude_credential_bridge.go | 3 +++ src/pkg/fleetreport/slogfeed/slogfeed.go | 19 ++++++++++++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-18107-claude-rebridge-selfreport.md diff --git a/changelog.d/fixed-18107-claude-rebridge-selfreport.md b/changelog.d/fixed-18107-claude-rebridge-selfreport.md new file mode 100644 index 0000000000..ecdbbba2d5 --- /dev/null +++ b/changelog.d/fixed-18107-claude-rebridge-selfreport.md @@ -0,0 +1 @@ +- Completed Claude credential re-bridges (a token refresh replacing an agent's shared-login symlink, repaired automatically) no longer file `unclassified_error_repeated` fleet self-reports; failed repairs still do. diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go index 1f36cb2096..4257ec3dde 100644 --- a/src/crux/agent/claude_credential_bridge.go +++ b/src/crux/agent/claude_credential_bridge.go @@ -567,6 +567,9 @@ func (m *Manager) rebridgeClaudeCredential(agentName, link, target, trigger stri if rep == nil { return nil } + // The fleet self-report feed deliberately ignores this exact message (a + // completed repair, expected in a burst at every token refresh): + // claudeCredentialRebridged in pkg/fleetreport/slogfeed. Keep them in sync. m.logger.Warn("claude credential bridge: a private credential file had replaced the shared-login symlink; re-bridged", "agent", agentName, "trigger", trigger, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 5a9507e578..3375f2902e 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -115,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) || claudeCredentialRebridged(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -143,6 +143,23 @@ func signedCommitOwnershipSkip(r slog.Record) bool { return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == "signed-commit reconciler: skipped; the PR keeps an unsigned commit" } +// claudeCredentialRebridgedMessage is the WARN crux/agent logs after it has +// SUCCESSFULLY restored a per-agent Claude credential symlink that a token +// refresh replaced (src/crux/agent/claude_credential_bridge.go). Keep the two +// strings in sync. +const claudeCredentialRebridgedMessage = "claude credential bridge: a private credential file had replaced the shared-login symlink; re-bridged" + +// claudeCredentialRebridged is the credential bridge's completed repair. Claude +// Code renames a fresh file over the shared-login symlink on every token +// refresh, every agent shares one access token and so refreshes within the same +// minute or two, and the watchdog checks each link every 5s: a burst of one +// re-bridge per agent is the designed recovery, already recorded in the audit +// log, not a defect (hivecommons/hive#18107). The WARN stays in local logs; a +// failed or refused repair logs a different message and stays reportable. +func claudeCredentialRebridged(r slog.Record) bool { + return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == claudeCredentialRebridgedMessage +} + // canceledError applies the existing cancellation exclusion to structured // errors too. An accompanying different error must remain reportable. func canceledError(r slog.Record, bound []slog.Attr) bool { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index d21d195f56..1c1a9c8131 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -84,6 +84,24 @@ func TestSignedCommitOwnershipSkipStaysInLogsWithoutFleetReport(t *testing.T) { } } +func TestClaudeCredentialRebridgeStaysInLogsWithoutFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + for range 6 { + logger.Warn(claudeCredentialRebridgedMessage, "agent", "reviewer", "trigger", "watchdog", "kept", "shared") + } + if len(*seen) != 0 { + t.Fatalf("completed re-bridges produced observations: %+v", *seen) + } + if n := strings.Count(out.String(), "re-bridged"); n != 6 { + t.Fatalf("local logs lost the re-bridge warning: got %d, want 6: %s", n, out.String()) + } + logger.Warn("claude credential bridge: repair failed", "agent", "reviewer", "error", "restore credential symlink: file exists") + logger.Error(claudeCredentialRebridgedMessage, "agent", "reviewer") + if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated { + t.Fatalf("failed repairs must remain reportable: %+v", *seen) + } +} + func TestCanceledStageDigestStaysLocal(t *testing.T) { logger, seen, out := newFeed(t) const msg = "stage labels: check digest failed" From 3b8ce0f420600c2e37d50da735a11593f9c9418e Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 13:53:53 -0500 Subject: [PATCH 161/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20omp=201?= =?UTF-8?q?8.19.0=20->=2018.20.0=20(#18064)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=18.20.0 SHA256_AMD64=c8694be8c6e6a1f45e3636dd8a8e9eaa78878ad629a89000b403ec5e456d3f89 SHA256_ARM64=8b5e197db7c58b7ebabf83c6f03597f89bae3a766a84c6481b3a942a125f5e22 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- src/Dockerfile | 6 +++--- src/Dockerfile.contributor | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index fd60ca9443..05b954dbe8 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -438,9 +438,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ # --- Layer 8.27: OMP CLI (Oh My Pi — backend: omp) --- # Mirrored from Dockerfile.contributor so hub agents can launch backend: omp. # Downloads are optional like Goose/agy, but checksum mismatches are fatal. -ARG OMP_VERSION=18.19.0 -ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 -ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 +ARG OMP_VERSION=18.20.0 +ARG OMP_SHA256_AMD64=c8694be8c6e6a1f45e3636dd8a8e9eaa78878ad629a89000b403ec5e456d3f89 +ARG OMP_SHA256_ARM64=8b5e197db7c58b7ebabf83c6f03597f89bae3a766a84c6481b3a942a125f5e22 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64|x86_64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 3e6703fdf8..dc3d62e858 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -492,9 +492,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # No credential is baked in: omp reads the selected model's provider # credential from its documented environment variables (ANTHROPIC_API_KEY, # OPENAI_API_KEY, …), which the Justfile's container arm already forwards. -ARG OMP_VERSION=18.19.0 -ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 -ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 +ARG OMP_VERSION=18.20.0 +ARG OMP_SHA256_AMD64=c8694be8c6e6a1f45e3636dd8a8e9eaa78878ad629a89000b403ec5e456d3f89 +ARG OMP_SHA256_ARM64=8b5e197db7c58b7ebabf83c6f03597f89bae3a766a84c6481b3a942a125f5e22 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ From 5385ef6804c890965e9bedbea7e096519b2d22a3 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 18:53:57 +0000 Subject: [PATCH 162/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.244.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.244.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 8 ++++++++ changelog.d/fixed-18059-contributor-unsearchable-warn.md | 1 - .../fixed-18093-knowledge-connector-not-loaded-badge.md | 1 - changelog.d/fixed-18105-ioscan-classifier-retry.md | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/fixed-18059-contributor-unsearchable-warn.md delete mode 100644 changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md delete mode 100644 changelog.d/fixed-18105-ioscan-classifier-retry.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b69a348f0e..ae387773f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.244.1) + +### Fixed + +- Contributor-lane auto-promotion now warns once per refused author and once per refused repository when GitHub's search cannot see a repository, instead of once per author × repository pair, so one invisible repository or one unsearchable author no longer floods WARN and self-reports as a repeated unclassified error (#18059). +- Knowledge Integrations: when the connector syncer is not running (`runtime: false`, hive booted with no `knowledge.connectors`), connector cards now show a `not loaded` badge with a restart hint and docs link instead of `pending`, and Connect / Check / Sync are disabled with the same hint rather than failing with 404 toasts; `pending` stays reserved for a loaded connector awaiting sign-in ([#18093](https://github.com/hivecommons/hive/issues/18093)). +- The ioscan classifier now retries a single transient model-endpoint failure (HTTP 429/5xx or a dropped connection) before applying its fail mode, so one gateway blip no longer passes untrusted text on unclassified or withholds it ([#18105](https://github.com/hivecommons/hive/issues/18105)). + ## 2026-11-13 (v5.244.0) ### Added diff --git a/changelog.d/fixed-18059-contributor-unsearchable-warn.md b/changelog.d/fixed-18059-contributor-unsearchable-warn.md deleted file mode 100644 index 75dd2ae587..0000000000 --- a/changelog.d/fixed-18059-contributor-unsearchable-warn.md +++ /dev/null @@ -1 +0,0 @@ -- Contributor-lane auto-promotion now warns once per refused author and once per refused repository when GitHub's search cannot see a repository, instead of once per author × repository pair, so one invisible repository or one unsearchable author no longer floods WARN and self-reports as a repeated unclassified error (#18059). diff --git a/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md b/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md deleted file mode 100644 index 319a50e047..0000000000 --- a/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge Integrations: when the connector syncer is not running (`runtime: false`, hive booted with no `knowledge.connectors`), connector cards now show a `not loaded` badge with a restart hint and docs link instead of `pending`, and Connect / Check / Sync are disabled with the same hint rather than failing with 404 toasts; `pending` stays reserved for a loaded connector awaiting sign-in ([#18093](https://github.com/hivecommons/hive/issues/18093)). diff --git a/changelog.d/fixed-18105-ioscan-classifier-retry.md b/changelog.d/fixed-18105-ioscan-classifier-retry.md deleted file mode 100644 index 133036f418..0000000000 --- a/changelog.d/fixed-18105-ioscan-classifier-retry.md +++ /dev/null @@ -1 +0,0 @@ -- The ioscan classifier now retries a single transient model-endpoint failure (HTTP 429/5xx or a dropped connection) before applying its fail mode, so one gateway blip no longer passes untrusted text on unclassified or withholds it ([#18105](https://github.com/hivecommons/hive/issues/18105)). From 9050ec6687de586e2807bb882f84ddb350fb6b58 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 14:01:47 -0500 Subject: [PATCH 163/223] =?UTF-8?q?=F0=9F=90=9B=20governor:=20hold=20kicks?= =?UTF-8?q?=20for=20agents=20out=20of=20provider=20quota=20(#18117)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(governor): hold kicks for agents out of provider quota The provider spend gate only fires for agents behind the hive's inference gateway, so an agent whose own provider refuses it for quota (Copilot's "You have exceeded your monthly quota") kept being kicked and sent review work every cycle, each turn ending at once, until review dispatches expired and PRs were parked for a human. Add a per-agent quota hold applied to the fully assembled kick list right after the spend gate: an agent whose QuotaExhausted flag is set has its kicks withheld, except one probe kick per governor.provider_budget.probe_interval_s. The probe time is stamped only when the kick is delivered, and forgotten once the flag clears. Healthy agents are never touched. The quota alert now says kicks are held and how often they are retried. Fixes #18112 Hive-Run: hivecommons/hive#18112 Hive-Plan: issue-18112 How to build it Hive-Spec: issue-18112#wanted-behavior Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * test(governor): cover the per-agent quota hold guard Mutation testing found the guard around gateKickMessagesForAgentQuota in runEvalCycle unreached by any test (main_eval_cycle.go:1111). Move it into agentQuotaHoldState.gate and test the empty-set, no-message and gating paths. The delivery hook now stamps probe times from the returned probe set, which is the same set of agents as before: a held agent's only kept kick is its probe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...fixed-quota-exhausted-agents-not-kicked.md | 1 + src/cmd/hive/eval_cycle_quota_hold.go | 137 +++++++++ src/cmd/hive/eval_cycle_quota_hold_test.go | 272 ++++++++++++++++++ src/cmd/hive/eval_cycle_seams.go | 9 +- src/cmd/hive/main_eval_cycle.go | 30 +- src/cmd/hive/provider_budget_alert_test.go | 19 +- .../sweeps-and-operations.md | 1 + 7 files changed, 459 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed-quota-exhausted-agents-not-kicked.md create mode 100644 src/cmd/hive/eval_cycle_quota_hold.go create mode 100644 src/cmd/hive/eval_cycle_quota_hold_test.go diff --git a/changelog.d/fixed-quota-exhausted-agents-not-kicked.md b/changelog.d/fixed-quota-exhausted-agents-not-kicked.md new file mode 100644 index 0000000000..cce7aae2e1 --- /dev/null +++ b/changelog.d/fixed-quota-exhausted-agents-not-kicked.md @@ -0,0 +1 @@ +- **Agents out of provider quota are no longer kicked every cycle** ([#18112](https://github.com/hivecommons/hive/issues/18112)). The provider spending-limit gate only saw agents behind the hive's inference gateway, so an agent whose own provider refused it — GitHub Copilot's "You have exceeded your monthly quota" — kept being kicked and sent review work all night, each turn ending at once, until its review dispatches expired and pull requests were parked for a human. The governor now holds kicks per agent while that agent's `QuotaExhausted` flag is set, lets one probe kick through per `governor.provider_budget.probe_interval_s` (default 30 minutes) to notice when the quota resets, and resumes normal kicking as soon as the flag clears. Other agents are never held, and the "provider quota exhausted" alert now says the kicks are held and how often they are retried. diff --git a/src/cmd/hive/eval_cycle_quota_hold.go b/src/cmd/hive/eval_cycle_quota_hold.go new file mode 100644 index 0000000000..f7b552c562 --- /dev/null +++ b/src/cmd/hive/eval_cycle_quota_hold.go @@ -0,0 +1,137 @@ +package main + +import ( + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/scheduler" +) + +// PER-AGENT PROVIDER QUOTA HOLD (hivecommons/hive#18112). +// +// The provider spend gate (gateKickMessagesForProviderBudget) only fires for +// agents behind the hive's own inference gateway: its latch is set by the +// gateway proxy, which never sees a backend that calls its provider directly +// (GitHub Copilot, for one). Those agents still raise crux.Process.QuotaExhausted +// from the pane ("You have exceeded your monthly quota"), but nothing withheld +// their kicks, so the governor kept kicking them — and sending them review +// work — every cycle, each turn ending at once on the same quota error. +// +// The hold is PER AGENT, unlike the spend gate, because the limit is on that +// agent's provider account, not on a key every agent shares: one agent out of +// quota must never hold any other agent. A held agent still gets one probe +// kick per governor.provider_budget.probe_interval_s, so the hive notices when +// the quota resets (the flag clears once the pane stops showing the message). + +// agentQuotaHoldState remembers, per quota-exhausted agent, when the last +// probe kick was actually delivered. Package-level (agentQuotaHold) for the +// same reason as providerBudgetProbe: runEvalCycle has no state of its own. +type agentQuotaHoldState struct { + mu sync.Mutex + lastProbe map[string]time.Time +} + +// MarkProbeDelivered records that a probe kick reached agent at the given +// time. Call it only after a kick is delivered: a withheld, deferred or failed +// kick learned nothing about the quota and must not push the next probe out. +func (s *agentQuotaHoldState) MarkProbeDelivered(agent string, at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if s.lastProbe == nil { + s.lastProbe = map[string]time.Time{} + } + s.lastProbe[agent] = at +} + +// Forget drops agent's probe time, so a later quota hit starts fresh. +func (s *agentQuotaHoldState) Forget(agent string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.lastProbe, agent) +} + +// LastProbe reports when a probe kick was last delivered to agent; the zero +// time means never, which makes the agent due a probe at once. +func (s *agentQuotaHoldState) LastProbe(agent string) time.Time { + s.mu.Lock() + defer s.mu.Unlock() + return s.lastProbe[agent] +} + +// quotaExhaustedAgents reads the per-agent quota flag from one status snapshot +// and forgets the probe time of every agent whose flag is clear, so its kicks +// flow normally and its next quota hit starts a fresh interval. nil entries +// are skipped. The returned set holds only the exhausted agents. +func (s *agentQuotaHoldState) quotaExhaustedAgents(statuses map[string]*crux.Process) map[string]bool { + exhausted := map[string]bool{} + for name, proc := range statuses { + if proc == nil { + continue + } + if proc.QuotaExhausted { + exhausted[name] = true + continue + } + s.Forget(name) + } + return exhausted +} + +// gate applies the per-agent quota hold to one cycle's kicks. It always reads +// the flags from statuses (so cleared agents forget their probe time), then +// returns messages unchanged when no agent is held or there is nothing to +// gate; otherwise it defers to gateKickMessagesForAgentQuota. +func (s *agentQuotaHoldState) gate(messages []scheduler.KickMessage, statuses map[string]*crux.Process, now time.Time, probeInterval time.Duration) (kept []scheduler.KickMessage, withheld []string, probes []string) { + exhausted := s.quotaExhaustedAgents(statuses) + if len(exhausted) == 0 || len(messages) == 0 { + return messages, nil, nil + } + return gateKickMessagesForAgentQuota(messages, func(agent string) bool { return exhausted[agent] }, s.LastProbe, now, probeInterval) +} + +// gateKickMessagesForAgentQuota withholds kicks for agents whose provider +// quota is exhausted, except for one probe kick per held agent once +// probeInterval has passed since that agent's last delivered probe (a zero +// lastProbe is due at once). Only the FIRST message for a probing agent is +// kept; the rest of that agent's messages are withheld this cycle. Agents +// that are not exhausted pass through untouched. Message order is preserved. +// +// withheld and probes name agents in message order; an agent appears in +// withheld once per withheld message, and in probes at most once. +func gateKickMessagesForAgentQuota(messages []scheduler.KickMessage, quotaExhausted func(agent string) bool, lastProbe func(agent string) time.Time, now time.Time, probeInterval time.Duration) (kept []scheduler.KickMessage, withheld []string, probes []string) { + decided := map[string]bool{} + for _, msg := range messages { + if !quotaExhausted(msg.Agent) { + kept = append(kept, msg) + continue + } + if !decided[msg.Agent] { + decided[msg.Agent] = true + if last := lastProbe(msg.Agent); last.IsZero() || now.Sub(last) >= probeInterval { + probes = append(probes, msg.Agent) + kept = append(kept, msg) + continue + } + } + withheld = append(withheld, msg.Agent) + } + return kept, withheld, probes +} + +// compactDuration renders an operator-facing interval without the zero tails +// of time.Duration.String: 30m, 2h, 1h30m, 45s. +func compactDuration(d time.Duration) string { + if d <= 0 { + return "0s" + } + s := d.Round(time.Second).String() + if strings.HasSuffix(s, "m0s") { + s = strings.TrimSuffix(s, "0s") + } + if strings.HasSuffix(s, "h0m") { + s = strings.TrimSuffix(s, "0m") + } + return s +} diff --git a/src/cmd/hive/eval_cycle_quota_hold_test.go b/src/cmd/hive/eval_cycle_quota_hold_test.go new file mode 100644 index 0000000000..77c938fa8f --- /dev/null +++ b/src/cmd/hive/eval_cycle_quota_hold_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "errors" + "log/slog" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/scheduler" +) + +// Tests for the per-agent provider quota hold (hivecommons/hive#18112). + +var quotaHoldNow = time.Date(2026, 11, 13, 9, 0, 0, 0, time.UTC) + +func quotaSet(agents ...string) func(string) bool { + set := map[string]bool{} + for _, a := range agents { + set[a] = true + } + return func(agent string) bool { return set[agent] } +} + +func probesAt(at map[string]time.Time) func(string) time.Time { + return func(agent string) time.Time { return at[agent] } +} + +func kickAgents(msgs []scheduler.KickMessage) []string { + out := []string{} + for _, m := range msgs { + out = append(out, m.Agent+":"+m.Message) + } + return out +} + +func TestQuotaHoldGate_NoHeldAgentsPassesThrough(t *testing.T) { + msgs := []scheduler.KickMessage{{Agent: "a", Message: "1"}, {Agent: "b", Message: "2"}} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet(), probesAt(nil), quotaHoldNow, 30*time.Minute) + if !reflect.DeepEqual(kept, msgs) { + t.Fatalf("kept = %v, want every message unchanged", kickAgents(kept)) + } + if len(withheld) != 0 || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want none", withheld, probes) + } +} + +func TestQuotaHoldGate_HeldAgentNotDueIsWithheld(t *testing.T) { + msgs := []scheduler.KickMessage{{Agent: "copilot", Message: "gov"}} + last := map[string]time.Time{"copilot": quotaHoldNow.Add(-10 * time.Minute)} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("copilot"), probesAt(last), quotaHoldNow, 30*time.Minute) + if len(kept) != 0 { + t.Fatalf("kept = %v, want the held agent's kick withheld", kickAgents(kept)) + } + if !reflect.DeepEqual(withheld, []string{"copilot"}) || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want [copilot] and no probe", withheld, probes) + } +} + +// A held agent that is due a probe gets exactly one kick; its other messages +// this cycle (for example a fix kick ahead of its governor kick) stay held. +func TestQuotaHoldGate_HeldAgentDueGetsOneProbe(t *testing.T) { + msgs := []scheduler.KickMessage{ + {Agent: "copilot", Message: "fix"}, + {Agent: "copilot", Message: "gov"}, + } + last := map[string]time.Time{"copilot": quotaHoldNow.Add(-30 * time.Minute)} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("copilot"), probesAt(last), quotaHoldNow, 30*time.Minute) + if want := []string{"copilot:fix"}; !reflect.DeepEqual(kickAgents(kept), want) { + t.Fatalf("kept = %v, want %v (first message only)", kickAgents(kept), want) + } + if !reflect.DeepEqual(probes, []string{"copilot"}) || !reflect.DeepEqual(withheld, []string{"copilot"}) { + t.Fatalf("probes=%v withheld=%v, want one probe and one withheld", probes, withheld) + } +} + +// One agent out of quota must not hold any other agent; order is kept. +func TestQuotaHoldGate_MixedHeldAndHealthyKeepsOrder(t *testing.T) { + msgs := []scheduler.KickMessage{ + {Agent: "healthy-1", Message: "a"}, + {Agent: "held", Message: "review"}, + {Agent: "healthy-2", Message: "b"}, + {Agent: "held", Message: "gov"}, + } + last := map[string]time.Time{"held": quotaHoldNow.Add(-time.Minute)} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("held"), probesAt(last), quotaHoldNow, 30*time.Minute) + if want := []string{"healthy-1:a", "healthy-2:b"}; !reflect.DeepEqual(kickAgents(kept), want) { + t.Fatalf("kept = %v, want %v", kickAgents(kept), want) + } + if !reflect.DeepEqual(withheld, []string{"held", "held"}) || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want [held held] and no probe", withheld, probes) + } +} + +func TestQuotaHoldGate_ZeroProbeTimeIsDueAtOnce(t *testing.T) { + msgs := []scheduler.KickMessage{ + {Agent: "healthy", Message: "a"}, + {Agent: "held", Message: "gov"}, + } + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("held"), probesAt(nil), quotaHoldNow, 30*time.Minute) + if want := []string{"healthy:a", "held:gov"}; !reflect.DeepEqual(kickAgents(kept), want) { + t.Fatalf("kept = %v, want %v", kickAgents(kept), want) + } + if !reflect.DeepEqual(probes, []string{"held"}) || len(withheld) != 0 { + t.Fatalf("probes=%v withheld=%v, want [held] probed at once", probes, withheld) + } +} + +func TestQuotaHoldState_MarkThenLastProbeThenForget(t *testing.T) { + var s agentQuotaHoldState + if got := s.LastProbe("a"); !got.IsZero() { + t.Fatalf("LastProbe on empty state = %v, want zero", got) + } + s.MarkProbeDelivered("a", quotaHoldNow) + if got := s.LastProbe("a"); !got.Equal(quotaHoldNow) { + t.Fatalf("LastProbe = %v, want %v", got, quotaHoldNow) + } + if got := s.LastProbe("b"); !got.IsZero() { + t.Fatalf("another agent's probe time = %v, want zero", got) + } + s.Forget("a") + if got := s.LastProbe("a"); !got.IsZero() { + t.Fatalf("LastProbe after Forget = %v, want zero", got) + } +} + +// When an agent's flag clears its probe time is forgotten, so its kicks flow +// normally and a later quota hit starts a fresh interval. +func TestQuotaHoldState_ClearedFlagForgetsProbe(t *testing.T) { + var s agentQuotaHoldState + s.MarkProbeDelivered("held", quotaHoldNow) + s.MarkProbeDelivered("recovered", quotaHoldNow) + got := s.quotaExhaustedAgents(map[string]*crux.Process{ + "held": {QuotaExhausted: true}, + "recovered": {QuotaExhausted: false}, + "gone": nil, + }) + if !reflect.DeepEqual(got, map[string]bool{"held": true}) { + t.Fatalf("exhausted = %v, want only held", got) + } + if !s.LastProbe("held").Equal(quotaHoldNow) { + t.Fatal("a still-held agent must keep its probe time") + } + if !s.LastProbe("recovered").IsZero() { + t.Fatal("a recovered agent's probe time must be forgotten") + } +} + +// gate is the eval cycle's entry point: it must return the cycle's messages +// untouched (same slice, no copy) when no agent is held or there is nothing to +// kick, still forget recovered agents, and gate only when both are non-empty. +func TestQuotaHoldState_GateShortCircuitsAndGates(t *testing.T) { + msgs := []scheduler.KickMessage{{Agent: "held", Message: "1"}, {Agent: "ok", Message: "2"}, {Agent: "held", Message: "3"}} + + t.Run("no held agent returns the same slice", func(t *testing.T) { + var s agentQuotaHoldState + s.MarkProbeDelivered("ok", quotaHoldNow) + kept, withheld, probes := s.gate(msgs, map[string]*crux.Process{"ok": {}}, quotaHoldNow, 30*time.Minute) + if len(kept) != len(msgs) || &kept[0] != &msgs[0] { + t.Fatalf("kept = %v, want the input slice itself", kickAgents(kept)) + } + if withheld != nil || probes != nil { + t.Fatalf("withheld=%v probes=%v, want nil", withheld, probes) + } + if !s.LastProbe("ok").IsZero() { + t.Fatal("a recovered agent's probe time must be forgotten even when nothing is gated") + } + }) + + t.Run("no messages returns the empty input", func(t *testing.T) { + var s agentQuotaHoldState + empty := []scheduler.KickMessage{} + kept, withheld, probes := s.gate(empty, map[string]*crux.Process{"held": {QuotaExhausted: true}}, quotaHoldNow, 30*time.Minute) + if kept == nil || len(kept) != 0 || withheld != nil || probes != nil { + t.Fatalf("kept=%v withheld=%v probes=%v, want the empty input and nothing else", kept, withheld, probes) + } + }) + + t.Run("one held agent is gated", func(t *testing.T) { + var s agentQuotaHoldState + s.MarkProbeDelivered("held", quotaHoldNow.Add(-time.Minute)) + kept, withheld, probes := s.gate(msgs, map[string]*crux.Process{"held": {QuotaExhausted: true}, "ok": {}}, quotaHoldNow, 30*time.Minute) + if got := kickAgents(kept); !reflect.DeepEqual(got, []string{"ok:2"}) { + t.Fatalf("kept = %v, want only ok:2", got) + } + if !reflect.DeepEqual(withheld, []string{"held", "held"}) || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want held twice and no probe", withheld, probes) + } + }) +} + +// The full cycle: withheld, then one probe after the interval, then withheld +// again on the next cycle once that probe was delivered. +func TestQuotaHoldGate_ProbeOncePerInterval(t *testing.T) { + var s agentQuotaHoldState + held := quotaSet("held") + msgs := []scheduler.KickMessage{{Agent: "held", Message: "gov"}} + interval := 30 * time.Minute + + s.MarkProbeDelivered("held", quotaHoldNow) + if kept, _, _ := gateKickMessagesForAgentQuota(msgs, held, s.LastProbe, quotaHoldNow.Add(5*time.Minute), interval); len(kept) != 0 { + t.Fatalf("inside the interval kept = %v, want none", kickAgents(kept)) + } + at := quotaHoldNow.Add(interval) + kept, _, probes := gateKickMessagesForAgentQuota(msgs, held, s.LastProbe, at, interval) + if len(kept) != 1 || !reflect.DeepEqual(probes, []string{"held"}) { + t.Fatalf("after the interval kept=%v probes=%v, want exactly one probe", kickAgents(kept), probes) + } + s.MarkProbeDelivered("held", at) + if kept, _, _ := gateKickMessagesForAgentQuota(msgs, held, s.LastProbe, at.Add(2*time.Minute), interval); len(kept) != 0 { + t.Fatalf("next cycle kept = %v, want the agent held again", kickAgents(kept)) + } +} + +// The probe time is recorded on delivery only: a probe whose send fails must +// not advance it, so the next cycle may probe again. +func TestQuotaHoldDispatch_ProbeRecordedOnDeliveryNotOnSendError(t *testing.T) { + for _, tc := range []struct { + name string + sendErr error + want bool + }{ + {"delivered", nil, true}, + {"send error", errors.New("agent CLI did not reach input prompt"), false}, + {"busy deferral", crux.ErrAgentBusy, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var s agentQuotaHoldState + exhausted := map[string]bool{"held": true} + r := newKickDispatchRecorder() + r.sendErr["held"] = tc.sendErr + deps := r.deps() + inner := deps.onDelivered + deps.onDelivered = func(msg scheduler.KickMessage, deliveredAt time.Time) { + inner(msg, deliveredAt) + if exhausted[msg.Agent] { + s.MarkProbeDelivered(msg.Agent, deliveredAt) + } + } + msgs, _, probes := gateKickMessagesForAgentQuota( + []scheduler.KickMessage{{Agent: "held", Message: "gov"}, {Agent: "healthy", Message: "gov"}}, + func(a string) bool { return exhausted[a] }, s.LastProbe, quotaHoldNow, 30*time.Minute) + if !reflect.DeepEqual(probes, []string{"held"}) { + t.Fatalf("probes = %v, want [held]", probes) + } + dispatchAgentKicks(msgs, false, deps, slog.Default()) + if got := !s.LastProbe("held").IsZero(); got != tc.want { + t.Fatalf("probe recorded = %v, want %v", got, tc.want) + } + if !s.LastProbe("healthy").IsZero() { + t.Fatal("a healthy agent's delivery must never record a probe time") + } + }) + } +} + +func TestCompactDuration(t *testing.T) { + for d, want := range map[time.Duration]string{ + 30 * time.Minute: "30m", + 2 * time.Hour: "2h", + 90 * time.Minute: "1h30m", + 45 * time.Second: "45s", + 90 * time.Second: "1m30s", + 0: "0s", + time.Hour + 30*time.Second: "1h0m30s", + } { + if got := compactDuration(d); got != want { + t.Errorf("compactDuration(%v) = %q, want %q", d, got, want) + } + } +} diff --git a/src/cmd/hive/eval_cycle_seams.go b/src/cmd/hive/eval_cycle_seams.go index 91925bc20e..e5d0e0af08 100644 --- a/src/cmd/hive/eval_cycle_seams.go +++ b/src/cmd/hive/eval_cycle_seams.go @@ -417,7 +417,11 @@ type providerBudgetAlert struct { // consulted the agent manager on the NOT-latched branch, and collecting agent // statuses every cycle while the provider is clipped would be work the old // code never did. Passing a thunk keeps that laziness exactly. -func decideProviderBudgetAlert(latched, suppress bool, cause string, since time.Time, rebuffs int, quotaReason func() string) providerBudgetAlert { +// +// quotaProbeInterval is how often an agent held for exhausted provider quota +// is probed (#18112); the quota banner names it so the operator knows the +// hive is holding those kicks and when it will retry them. +func decideProviderBudgetAlert(latched, suppress bool, cause string, since time.Time, rebuffs int, quotaProbeInterval time.Duration, quotaReason func() string) providerBudgetAlert { if latched { state := "agent kicks suspended" if !suppress { @@ -431,7 +435,8 @@ func decideProviderBudgetAlert(latched, suppress bool, cause string, since time. return providerBudgetAlert{Message: msg, Cause: msg} } if reason := quotaReason(); reason != "" { - return providerBudgetAlert{Message: "provider quota exhausted — " + reason} + return providerBudgetAlert{Message: fmt.Sprintf("provider quota exhausted — %s; their kicks are held and retried every %s", + reason, compactDuration(quotaProbeInterval))} } return providerBudgetAlert{Clear: true} } diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 0c56eb0086..26e660c119 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -91,6 +91,11 @@ var providerBudgetNotify governor.ProviderBudgetNotifyState // providerBudgetNotify: runEvalCycle has no state of its own. var providerBudgetProbe governor.ProviderBudgetProbeState +// agentQuotaHold remembers, per agent out of provider quota, when its last +// probe kick was delivered (hivecommons/hive#18112). Package-level for the +// same reason as providerBudgetProbe. +var agentQuotaHold agentQuotaHoldState + // governorKickFairness remembers, per agent, how many cycles in a row the // governor kick was deferred behind a delivered review-fix kick // (hivecommons/hive#15166). Package-level for the same reason as @@ -886,7 +891,7 @@ func runEvalCycle( suppressKicks := governor.ProviderBudgetSuppresses(providerBudgetLatched, providerBudgetProbe.Freshest(providerBudgetLastRebuff), time.Now(), providerBudgetProbeInterval) budgetAlert := decideProviderBudgetAlert(providerBudgetLatched, suppressKicks, - providerBudgetCause, providerBudgetSince, providerBudgetRebuffs, + providerBudgetCause, providerBudgetSince, providerBudgetRebuffs, providerBudgetProbeInterval, func() string { return hub.QuotaExhaustedAgentReason(hub.QuotaExhaustedProcessCount(agentMgr.AllStatuses())) }) @@ -1093,6 +1098,23 @@ func runEvalCycle( "last_rebuff", providerBudgetLastRebuff, "probe_interval", providerBudgetProbeInterval) } messages = kickGate.Kept + // PER-AGENT PROVIDER QUOTA HOLD (#18112). The spend gate above only sees + // agents behind the hive's inference gateway; an agent whose own provider + // refuses it for quota (Copilot's "monthly quota") is held here, per + // agent, from the QuotaExhausted flag every backend reports. Review-swarm + // kicks are in this list too, so a held reviewer gets no review work. + quotaProbeNow := map[string]bool{} + { + kept, quotaWithheld, quotaProbes := agentQuotaHold.gate(messages, agentMgr.AllStatuses(), time.Now(), providerBudgetProbeInterval) + if len(quotaWithheld) > 0 || len(quotaProbes) > 0 { + logger.Warn("provider quota exhausted: withholding agent kicks", + "withheld", quotaWithheld, "probe", quotaProbes, "probe_interval", providerBudgetProbeInterval) + } + for _, agent := range quotaProbes { + quotaProbeNow[agent] = true + } + messages = kept + } // Self-upgrade drain (#11876): a hub-instructed upgrade is waiting for // in-flight kicks to end, so starting new turns would only feed the // restart. Withheld kicks stay due and go out from the new process. @@ -1207,6 +1229,12 @@ func runEvalCycle( }, onDelivered: func(msg scheduler.KickMessage, deliveredAt time.Time) { deliveredKicks = append(deliveredKicks, msg) + if quotaProbeNow[msg.Agent] { + // The only kick a held agent can receive is its probe; + // stamp it on delivery so a failed or deferred probe + // is retried next cycle (#18112). + agentQuotaHold.MarkProbeDelivered(msg.Agent, deliveredAt) + } gov.RecordKickForRepoAt(msg.Agent, msg.Repo, deliveredAt) dashSrv.AuditLog("governor", "kick", "trigger=governor-eval", msg.Agent) diff --git a/src/cmd/hive/provider_budget_alert_test.go b/src/cmd/hive/provider_budget_alert_test.go index bcfb10685f..878d96bb04 100644 --- a/src/cmd/hive/provider_budget_alert_test.go +++ b/src/cmd/hive/provider_budget_alert_test.go @@ -23,7 +23,7 @@ func neverCalled(t *testing.T) func() string { } func TestDecideProviderBudgetAlertLatchedSuspended(t *testing.T) { - got := decideProviderBudgetAlert(true, true, "spend cap hit", time.Now(), 1, neverCalled(t)) + got := decideProviderBudgetAlert(true, true, "spend cap hit", time.Now(), 1, 30*time.Minute, neverCalled(t)) if got.Clear { t.Fatal("a held latch must not clear the banner") @@ -40,7 +40,7 @@ func TestDecideProviderBudgetAlertLatchedSuspended(t *testing.T) { } func TestDecideProviderBudgetAlertLatchedProbing(t *testing.T) { - got := decideProviderBudgetAlert(true, false, "spend cap hit", time.Now(), 1, neverCalled(t)) + got := decideProviderBudgetAlert(true, false, "spend cap hit", time.Now(), 1, 30*time.Minute, neverCalled(t)) if !strings.Contains(got.Message, "probing with a single agent kick") { t.Fatalf("a stale latch probes rather than staying muted, got %q", got.Message) @@ -54,7 +54,7 @@ func TestDecideProviderBudgetAlertLatchedProbing(t *testing.T) { // between "one call was refused" and "this has been refusing since Tuesday". func TestDecideProviderBudgetAlertRepeatedRebuffsAreQuantified(t *testing.T) { since := time.Date(2026, 9, 16, 14, 30, 0, 0, time.UTC) - got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 47, neverCalled(t)) + got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 47, 30*time.Minute, neverCalled(t)) if !strings.Contains(got.Message, "47 refused calls") { t.Fatalf("repeated rebuffs must be counted, got %q", got.Message) @@ -66,7 +66,7 @@ func TestDecideProviderBudgetAlertRepeatedRebuffsAreQuantified(t *testing.T) { func TestDecideProviderBudgetAlertSingleRebuffIsNotQuantified(t *testing.T) { since := time.Date(2026, 9, 16, 14, 30, 0, 0, time.UTC) - got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 1, neverCalled(t)) + got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 1, 30*time.Minute, neverCalled(t)) if strings.Contains(got.Message, "refused calls") { t.Fatalf("a single rebuff must not be reported as a run of them, got %q", got.Message) @@ -80,7 +80,7 @@ func TestDecideProviderBudgetAlertSingleRebuffIsNotQuantified(t *testing.T) { // branch: the provider is not refusing on money, but agents are individually // out of quota, which is a different cause and must not be silent. func TestDecideProviderBudgetAlertQuotaExhaustedFallback(t *testing.T) { - got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, + got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, 30*time.Minute, func() string { return "3 agents out of quota" }) if got.Clear { @@ -89,13 +89,18 @@ func TestDecideProviderBudgetAlertQuotaExhaustedFallback(t *testing.T) { if !strings.Contains(got.Message, "provider quota exhausted") || !strings.Contains(got.Message, "3 agents out of quota") { t.Fatalf("quota exhaustion must name itself and its reason, got %q", got.Message) } + // #18112: the banner says the hive is holding those agents' kicks and + // when it retries them, not just that the quota ran out. + if !strings.Contains(got.Message, "their kicks are held and retried every 30m") { + t.Fatalf("quota exhaustion must say kicks are held and the probe interval, got %q", got.Message) + } if got.Cause != "" { t.Fatalf("only the latched branch replaces the cause, got %q", got.Cause) } } func TestDecideProviderBudgetAlertHealthyClears(t *testing.T) { - got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, func() string { return "" }) + got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, 30*time.Minute, func() string { return "" }) if !got.Clear { t.Fatal("a healthy provider with no exhausted quota must clear the banner, or it sticks forever") @@ -121,7 +126,7 @@ func TestDecideProviderBudgetAlertNeverRaisesAndClears(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := decideProviderBudgetAlert(tc.latched, tc.suppress, "cause", time.Now(), 1, + got := decideProviderBudgetAlert(tc.latched, tc.suppress, "cause", time.Now(), 1, 30*time.Minute, func() string { return tc.quota }) if got.Message != "" && got.Clear { t.Fatalf("must never raise and clear in the same cycle: %+v", got) diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index e9e558eb20..1adeee5c39 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -916,6 +916,7 @@ governor: ``` Lower it to resume sooner after a reset at the cost of a rebuffed run per probe; raise it to waste less while noticing later. +- An agent whose **own** provider refuses it for quota — for example GitHub Copilot's "You have exceeded your monthly quota", which never passes through the hive's inference gateway — is held per agent ([#18112](https://github.com/hivecommons/hive/issues/18112)). Every backend reports the pane's quota message as the agent's `QuotaExhausted` flag; while it is set, the governor withholds that agent's kicks, including review-swarm review and fix kicks, so a held reviewer is not sent review work that would only expire. Once per `governor.provider_budget.probe_interval_s` (the same setting as above) one kick is let through as a probe, and the interval restarts only when that probe is actually delivered; a failed or deferred probe is retried on the next cycle. When the pane stops showing the quota message the flag clears and the agent's kicks flow normally from the next cycle. One agent out of quota never holds another, and manual kicks from the dashboard are not held. The dashboard alert reads `provider quota exhausted — N agent(s) out of provider quota; their kicks are held and retried every 30m`, and the log shows `provider quota exhausted: withholding agent kicks` with the withheld and probed agents. ## Merge-hotspot advisor (`governor.hotspot_*`) From e527ba93b98fff8c7290b12867614618ee97d61c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:08:58 +0000 Subject: [PATCH 164/223] fix: stop repeating close-on-merge warning for unreadable PRs (#18103) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- ...ixed-18084-close-on-merge-unreadable-pr.md | 1 + src/pkg/github/close_on_merge.go | 16 ++++++++++++++ src/pkg/github/close_on_merge_paths_test.go | 21 +++++++++++++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-18084-close-on-merge-unreadable-pr.md diff --git a/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md b/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md new file mode 100644 index 0000000000..76d3978235 --- /dev/null +++ b/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md @@ -0,0 +1 @@ +- Close-on-merge no longer re-warns every tick for merged PRs it cannot read (404/403); it logs once per recheck window. diff --git a/src/pkg/github/close_on_merge.go b/src/pkg/github/close_on_merge.go index 239bc97fda..08cf0ecbb8 100644 --- a/src/pkg/github/close_on_merge.go +++ b/src/pkg/github/close_on_merge.go @@ -192,6 +192,17 @@ func closeOnMergeClaimSettled(claim IssueClaim, now time.Time, recheck time.Dura return now.Sub(claim.CloseOnMergeSettledAt) < recheck } +// closeOnMergePRUnreadable reports a terminal read failure: the PR is gone or +// this identity cannot see it (404/403), so retrying every tick is noise. +func closeOnMergePRUnreadable(err error) bool { + var ghErr *gh.ErrorResponse + if !errors.As(err, &ghErr) || ghErr.Response == nil { + return false + } + code := ghErr.Response.StatusCode + return code == http.StatusNotFound || code == http.StatusForbidden +} + // closeOnMergeClaimMergedAt is the instant the sweep treats as the claim's // merge time for MergedAfter filtering: MergedAt, else the first observation. func closeOnMergeClaimMergedAt(claim IssueClaim) time.Time { @@ -255,6 +266,11 @@ func (c *Client) CloseOnMergeLedgerSweep(ctx context.Context, ledger *ClaimLedge if opts.Logger != nil { opts.Logger.Warn("close-on-merge: could not read merged PR", "repo", claim.PRRepo, "pr", claim.PRNumber, "error", err) } + // 404/403 will not heal on the next tick: remember it so the + // warning repeats at most once per recheck window. + if closeOnMergePRUnreadable(err) && ledger.MarkCloseOnMergeSettled(claim.Repo, claim.Issue, claim.PRNumber, now) { + sweep.Marked++ + } continue } res := c.closeIssueForMergedPRClaim(ctx, claim.Repo, claim.Issue, pr, true, opts) diff --git a/src/pkg/github/close_on_merge_paths_test.go b/src/pkg/github/close_on_merge_paths_test.go index eacbbe2ba5..d9cec8f187 100644 --- a/src/pkg/github/close_on_merge_paths_test.go +++ b/src/pkg/github/close_on_merge_paths_test.go @@ -107,6 +107,10 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) return } + if strings.HasSuffix(r.URL.Path, "/404") { + w.WriteHeader(http.StatusNotFound) + return + } _ = json.NewEncoder(w).Encode(map[string]any{ "number": 42, "title": "fix", "body": "Fixes #7", "merged": true, "merged_at": "2026-10-09T12:00:00Z", "merge_commit_sha": "abcdef0123456789", @@ -143,6 +147,8 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { {Repo: "o/r", Issue: 9, PRNumber: 44, PRRepo: "o/r", MergedPR: true, Reference: true}, // Merged claim whose PR cannot be read: logged and skipped. {Repo: "o/r", Issue: 10, PRNumber: 500, PRRepo: "o/r", MergedPR: true}, + // Merged claim whose PR is 404: warned once, then settled. + {Repo: "o/r", Issue: 12, PRNumber: 404, PRRepo: "o/r", MergedPR: true}, // Merged claim missing a PR number: skipped. {Repo: "o/r", Issue: 11, PRNumber: 0, PRRepo: "o/r", MergedPR: true}, } { @@ -170,8 +176,19 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { if len(patched) != 1 { t.Fatalf("patched = %v, want one close", patched) } - if len(prGets) != 2 { - t.Fatalf("PR GETs = %v, want /pulls/42 and /pulls/500 only", prGets) + if len(prGets) != 3 { + t.Fatalf("PR GETs = %v, want /pulls/42, /pulls/500 and /pulls/404 only", prGets) + } + // Second sweep: the 404 claim is settled and not re-read; the 500 one is. + prGets = nil + c.CloseOnMergeForLedger(context.Background(), ledger, CloseOnMergeOptions{ + Identity: HiveIdentity{AppLogin: "hive[bot]"}, + Logger: logger, + }) + for _, p := range prGets { + if strings.HasSuffix(p, "/404") { + t.Fatalf("404 PR re-read on second sweep: %v", prGets) + } } if !strings.Contains(logBuf.String(), "could not read merged PR") { t.Fatalf("unreadable PR was not logged: %s", logBuf.String()) From 3b1eaeda635b2733ad3ef3e6589b9b1f917c2098 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 14:11:09 -0500 Subject: [PATCH 165/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20codex?= =?UTF-8?q?=200.167.0=20->=200.168.0=20(#18062)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=0.168.0 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 05b954dbe8..27866c4f76 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -324,7 +324,7 @@ RUN timeout 120s npm install -g --ignore-scripts --no-fund --no-audit \ # dashboard's codex model discovery (`codex app-server` model/list — pure # local stdio, no network, unaffected by the proxy CA) falls back to the # static list, and non-codex backends are unaffected. -ARG CODEX_VERSION=0.167.0 +ARG CODEX_VERSION=0.168.0 ARG CLINE_VERSION=3.0.74 # npm can briefly advertise AWS SDK versions before all registry replicas serve # them. Wait at most three minutes between attempts and revalidate metadata. diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index dc3d62e858..9013899341 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -262,7 +262,7 @@ ARG KILO_CLI_VERSION=7.5.6 # Same ARG name and value as src/Dockerfile so bin/test_backend_smoke.sh can # assert the two pins agree — this line was previously an inline hardcode that # could (and did) drift from the main image's CODEX_VERSION unnoticed. -ARG CODEX_VERSION=0.167.0 +ARG CODEX_VERSION=0.168.0 RUN npm install -g --ignore-scripts --no-fund --no-audit \ @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ @github/copilot@${COPILOT_VERSION} \ From 9caf088dcd2951863ace9f5b6194468a5769f916 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 19:26:04 +0000 Subject: [PATCH 166/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.245.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.245.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 11 +++++++++++ changelog.d/added-17888-ship-spektacular.md | 1 - changelog.d/fixed-18107-claude-rebridge-selfreport.md | 1 - .../fixed-18110-navbar-agents-summary-button.md | 1 - 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/added-17888-ship-spektacular.md delete mode 100644 changelog.d/fixed-18107-claude-rebridge-selfreport.md delete mode 100644 changelog.d/fixed-18110-navbar-agents-summary-button.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ae387773f5..70ad82bdc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.245.0) + +### Added + +- The hive image now ships the `spektacular` CLI (pinned 0.31.0, checksum-verified, amd64 and arm64), so the Inbox → RFC destination runs `spektacular voice --publish` in-pod without an agent fallback (#17888). + +### Fixed + +- Completed Claude credential re-bridges (a token refresh replacing an agent's shared-login symlink, repaired automatically) no longer file `unclassified_error_repeated` fleet self-reports; failed repairs still do. +- Dashboard top bar: when the centre column is narrow, the agent run-order strip's summary pill is now a real button that opens the Agents section (click, Enter or Space), and it shows how many scheduled agents are in the run order ("9 agents", "1 agent") instead of a count capped at 3; the misleading "▾" caret is gone ([#18110](https://github.com/hivecommons/hive/issues/18110)). + ## 2026-11-13 (v5.244.1) ### Fixed diff --git a/changelog.d/added-17888-ship-spektacular.md b/changelog.d/added-17888-ship-spektacular.md deleted file mode 100644 index db2fb2c358..0000000000 --- a/changelog.d/added-17888-ship-spektacular.md +++ /dev/null @@ -1 +0,0 @@ -- The hive image now ships the `spektacular` CLI (pinned 0.31.0, checksum-verified, amd64 and arm64), so the Inbox → RFC destination runs `spektacular voice --publish` in-pod without an agent fallback (#17888). diff --git a/changelog.d/fixed-18107-claude-rebridge-selfreport.md b/changelog.d/fixed-18107-claude-rebridge-selfreport.md deleted file mode 100644 index ecdbbba2d5..0000000000 --- a/changelog.d/fixed-18107-claude-rebridge-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- Completed Claude credential re-bridges (a token refresh replacing an agent's shared-login symlink, repaired automatically) no longer file `unclassified_error_repeated` fleet self-reports; failed repairs still do. diff --git a/changelog.d/fixed-18110-navbar-agents-summary-button.md b/changelog.d/fixed-18110-navbar-agents-summary-button.md deleted file mode 100644 index 7c48d4aa50..0000000000 --- a/changelog.d/fixed-18110-navbar-agents-summary-button.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard top bar: when the centre column is narrow, the agent run-order strip's summary pill is now a real button that opens the Agents section (click, Enter or Space), and it shows how many scheduled agents are in the run order ("9 agents", "1 agent") instead of a count capped at 3; the misleading "▾" caret is gone ([#18110](https://github.com/hivecommons/hive/issues/18110)). From 811b61e8de7e1030a83e288302cf8a243ad12dd3 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 14:26:22 -0500 Subject: [PATCH 167/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20Claude=20subsc?= =?UTF-8?q?ription=20usage=20fuel=20gauge=20on=20the=20Overview=20(v5)=20(?= =?UTF-8?q?#18118)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: add dashboard.subscription_gauge display flag Add an owner-controlled, display-only dashboard.subscription_gauge flag, off by default and omitted from YAML when off. It is read and saved through the existing owner-only /api/config/dashboard settings path (audited like the other dashboard settings) and surfaced to the page as features.subscription_gauge in the status frame. It carries no usage data and changes nothing about measurement or rotation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✨ rotation: include plan_type in headroom JSON when known Headroom.MarshalJSON dropped PlanType. Emit it as an optional plan_type, only when a probe reported one, so the dashboard can show the plan name. Readings without a plan (Claude's today) serialize exactly as before; the contributor guard files use their own ContributorReading shape and are unaffected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✨ dashboard: header-only usage gauge switch in Settings → Dashboard The Claude usage gauge will be on the Overview by default for the owner, so the dashboard flag now only adds the header chip. Rename it to dashboard.subscription_gauge_header (config, settings API, status flag and tests) and add an "Also show the usage gauge in the header" switch under Settings → Dashboard → Dashboard runtime, with owner-only, not-the-token- budget help text and a settings-search entry. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WcWm7qSas33JDrgas6zFWf Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✨ dashboard: subscription gauge view-model with unit tests Add subscriptionGaugeModel, the single pure decision point for the Claude usage gauge: it turns a /api/providers/headroom response into one dial per plan limit (longest first), the needle on the limit closest to empty, warning/critical status at 25%/10% left, out-of-date from the reading's captured_at, and a no-reading reason per probe cause. It handles both the builtin and the ccleft reading shapes and ignores credit pools and other providers. node:test coverage in subscription_gauge.test.js. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WcWm7qSas33JDrgas6zFWf Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✨ dashboard: Claude usage fuel gauge on the Overview Show the owner a fuel gauge on the Overview page by default whenever the hive uses Claude: one E-to-F dial per plan limit with percent left, reset time and reading age, or the no-reading reason. It reads the stored /api/providers/headroom readings once a minute while visible, only after the viewer is known to be the owner, so it adds no calls to Claude and never sends a non-owner a 403. An owner-only "Show Claude usage gauge" checkbox in Overview settings hides it per browser. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WcWm7qSas33JDrgas6zFWf Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✨ dashboard: opt-in Claude usage chip in the header With dashboard.subscription_gauge_header on, the owner also gets a header chip on every page: a compact fuel dial whose needle shows the limit closest to empty, "Claude N%" (with " · 5-hour" or " · model" when that is not the weekly limit), a tooltip listing every limit, and a click that opens the gauge on the Overview. It reuses the Overview gauge's readings, so it makes no request of its own; it stays hidden for everyone else. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WcWm7qSas33JDrgas6zFWf Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 📖 docs: Claude usage gauge, header switch and owner-only headroom API Document the Overview Claude usage gauge, how to hide it, and the dashboard.subscription_gauge_header switch (dashboard.md, a new Dashboard entry in settings.md, the operator reference), add GET/PUT /api/config/dashboard to the API reference, correct /api/providers/headroom to owner-only with plan_type, and add the changelog fragment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WcWm7qSas33JDrgas6zFWf Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 📖 docs: re-anchor api-reference citations after server.go grew Adding FrontendFeatures.SubscriptionGaugeHeader shifted the route registrations in pkg/dashboard/server.go by four lines. Rewritten with src/scripts/check-docs-citations.py --fix (16 rewritten, 0 unresolved). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WcWm7qSas33JDrgas6zFWf Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-subscription-usage-gauge.md | 1 + src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/configuration.md | 2 + .../api-reference/cost-tokens-telemetry.md | 2 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/dashboard.md | 23 ++ src/docs/operator-reference/README.md | 1 + src/docs/settings.md | 6 + src/pkg/config/config.go | 7 + src/pkg/dashboard/api_dashboard_config.go | 62 +-- .../dashboard/api_dashboard_config_test.go | 94 +++++ src/pkg/dashboard/server.go | 4 + src/pkg/dashboard/static-src/css/overview.css | 34 ++ src/pkg/dashboard/static-src/css/topbar.css | 6 + .../static-src/html/overview-governor.html | 1 + src/pkg/dashboard/static-src/html/topbar.html | 1 + .../dashboard/static-src/js/config-state.js | 1 + .../dashboard/static-src/js/config-tabs.js | 1 + src/pkg/dashboard/static-src/js/gh-auth.js | 1 + src/pkg/dashboard/static-src/js/overview.js | 13 +- src/pkg/dashboard/static-src/js/refresh.js | 1 + .../static-src/js/subscription-gauge.js | 300 +++++++++++++++ src/pkg/dashboard/static-src/manifest.txt | 1 + src/pkg/dashboard/static/index.html | 359 +++++++++++++++++- src/pkg/dashboard/status_builder.go | 5 +- src/pkg/dashboard/status_builder_test.go | 31 ++ src/pkg/dashboard/subscription_gauge.test.js | 171 +++++++++ .../dashboard/subscription_gauge_chip_test.go | 55 +++ .../subscription_gauge_overview_test.go | 81 ++++ src/pkg/rotation/rotation.go | 4 + src/pkg/rotation/rotation_test.go | 36 ++ 33 files changed, 1283 insertions(+), 53 deletions(-) create mode 100644 changelog.d/added-subscription-usage-gauge.md create mode 100644 src/pkg/dashboard/static-src/js/subscription-gauge.js create mode 100644 src/pkg/dashboard/subscription_gauge.test.js create mode 100644 src/pkg/dashboard/subscription_gauge_chip_test.go create mode 100644 src/pkg/dashboard/subscription_gauge_overview_test.go diff --git a/changelog.d/added-subscription-usage-gauge.md b/changelog.d/added-subscription-usage-gauge.md new file mode 100644 index 0000000000..87e70297fb --- /dev/null +++ b/changelog.d/added-subscription-usage-gauge.md @@ -0,0 +1 @@ +- Dashboard: an owner-only fuel gauge on the Overview now shows how much of the hive's Claude plan usage is left in each limit (weekly, plus a 5-hour or model-specific limit when the plan has one) and when each resets, turning amber at 25% left and red at 10%. It is on by default whenever the hive uses Claude, can be hidden from the Overview settings, and can also be shown as a header chip on every page with `dashboard.subscription_gauge_header`. It displays the usage readings hive already makes, so it adds no calls to Claude and changes nothing about measurement or provider rotation; `/api/providers/headroom` readings now also carry `plan_type` when a provider reports one. diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index a332a30544..08de2de8c9 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1708` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1709` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1710` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1712` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1713` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1714` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index c11dea097d..a5a707e5e6 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -10,6 +10,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/config` | Dashboard auth/session | Config | `pkg/dashboard/api.go:50` | | `GET` | `/api/config/triggers` | Verified owner only | Ordered Event triggers, opaque `version`/ETag, existing agents and managed repository values/labels; applies on next evaluation without restart | `pkg/dashboard/api_event_triggers.go` | | `PUT` | `/api/config/triggers` | Verified owner only | Compile-to-bool and durably compare-and-save `{rules:[{name,agent,expr,priority}],preset?:{index,kind,repo}}`; exact `If-Match` required (428 missing, 412 stale), validation 400, persistence failure 500 leaves live rules unchanged. No enabled field or source/webhook installation | `pkg/dashboard/api_event_triggers.go` | +| `GET` | `/api/config/dashboard` | Owner only | Dashboard runtime settings (`dashboard.*`): port, agent poll interval, `experiment_lab`, `subscription_gauge_header`, hub-proxied auth, whether an auth token is configured (never the token), access allowlist, issue bands and stage-tracker toast timings | `pkg/dashboard/api.go:213` | +| `PUT` | `/api/config/dashboard` | Owner only | Update any subset of those fields (an absent field is unchanged), including `subscription_gauge_header` (adds the owner-only Claude usage chip to the header); validated, saved and audit-logged as `config_dashboard_settings` | `pkg/dashboard/api.go:214` | | `GET` | `/api/config/dashboard/theme` | Owner only | Dashboard theme selection, overrides, effective theme, and built-in catalog | `pkg/dashboard/api.go:59` | | `PUT` | `/api/config/dashboard/theme` | Owner only | Persist `dashboard.theme` and `dashboard.theme_overrides` with validation, CSS/background guardrails, and audit logging | `pkg/dashboard/api.go:60` | | `GET` | `/api/config/download` | Owner only | Config Download | `pkg/dashboard/api.go:51` | diff --git a/src/docs/api-reference/cost-tokens-telemetry.md b/src/docs/api-reference/cost-tokens-telemetry.md index e80975236d..3628745685 100644 --- a/src/docs/api-reference/cost-tokens-telemetry.md +++ b/src/docs/api-reference/cost-tokens-telemetry.md @@ -16,5 +16,5 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/cost/history` | Dashboard auth/session | Cost History | `pkg/dashboard/api.go:169` | | `GET` | `/api/trend/history` | Dashboard auth/session | Trend History | `pkg/dashboard/api.go:172` | | `GET` | `/api/timeseries` | Dashboard auth/session | Time Series | `pkg/dashboard/api.go:174` | -| `GET` | `/api/providers/headroom` | Dashboard auth/session | Providers Headroom | `pkg/dashboard/api.go:64` | +| `GET` | `/api/providers/headroom` | Owner only | Last stored per-provider usage readings (`providers[]` with `limits[]`, `captured_at`, `stale`, `probe_error_cause`, and `plan_type` when a provider reports one), from provider rotation or the default publish-only reading; serving it never triggers a new measurement. Feeds the Overview Claude usage gauge and the rotation settings table | `pkg/dashboard/api.go:64` | | `GET` | `/api/budget/history` | Dashboard auth/session | Budget History | `pkg/dashboard/api.go:171` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 5d10bcbcba..41b4d16486 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1715` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1719` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 7d08eb0603..5ac2c544e1 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1688` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1689` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1690` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1696` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1701` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1702` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1703` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1704` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1705` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1706` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1707` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1692` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1693` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1694` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1700` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1705` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1706` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1707` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1708` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1709` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1710` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1711` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 714349077f..e67c5dc458 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1778` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1782` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/dashboard.md b/src/docs/dashboard.md index 75f6126847..f67af70dbd 100644 --- a/src/docs/dashboard.md +++ b/src/docs/dashboard.md @@ -125,6 +125,29 @@ honoured as a deprecated alias for one release: it enables the panel, logs a one-line deprecation at config load, and is folded into `dashboard.experiment_lab` so the next config save writes the documented key. +### Claude usage gauge + +When the hive uses Claude, the owner sees a fuel gauge at the top of the +**Overview** card: one dial for each usage limit the Claude plan reports (the +weekly limit, plus a 5-hour or model-specific weekly limit when the plan has +one), each with the percentage left and when it resets. A dial turns amber at +25% left or less and red at 10% or less, and the gauge says how old the reading +is and greys out when it is more than 15 minutes old. When there is no reading, +it says so and why (for example, the hive is not signed in to Claude) instead +of showing an empty dial. + +The gauge shows the measurements hive already makes on its regular 5-minute +schedule, through provider rotation or the usage reading that runs by default +without it. It adds no calls to Claude and never moves or pauses an agent. It +reflects the Claude account the hive itself is signed in with, and it is the +Claude plan's usage, not the hive's own token budget. + +Only the owner sees it. To hide it, untick **Show Claude usage gauge** in the +Overview settings (remembered per browser). To also show it as a small chip in +the header on every page, turn on **Also show the usage gauge in the header** +(`dashboard.subscription_gauge_header`) under Settings → Dashboard; clicking the +chip opens the gauge on the Overview. + ## Governor card The dashboard **Governor** card summarizes queue depth, operating mode, budget diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 48998e010e..7c0ce4320d 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -294,6 +294,7 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `dashboard.authorized_users` | Empty means no per-user direct-route allowlist. | Entries can be `user` or `user:role`; roles are `read`, `read-write`, `merger`, `owner`. | | `dashboard.public_url` | Empty means OAuth redirect URIs (Linear agent install, OpenRouter funding) fall back to `hub.dashboard_url`, then the request's forwarded/`Host` origin. | Set to this dashboard's externally reachable origin (`https://hive.example.com`, no path/query) on a standalone hive whose callback is published on a different hostname or whose ingress rewrites `Host`. Must be an absolute `http(s)://` origin; a trailing slash is trimmed and anything else fails config load. See [linear-agent.md](../linear-agent.md#setup). | | `dashboard.experiment_lab` | Off by default. | Hides the Experiment Lab (formerly Strategy Lab) while that surface is being reworked. Set `dashboard.experiment_lab: true` to show the Experiment Lab section, sidebar nav entry, and Nous status controls. | +| `dashboard.subscription_gauge_header` | Off by default. | Also shows the owner-only Claude usage fuel gauge as a chip in the dashboard header on every page. The gauge is on the Overview by default whenever the hive uses Claude and needs no setting; this key only adds the header chip. Display only: it reads the existing usage readings and changes nothing about measurement or rotation. See [Claude usage gauge](../dashboard.md#claude-usage-gauge). | | `dashboard.strategy_lab` | Off by default. **Deprecated** alias of `dashboard.experiment_lab`, kept for one release (#11695). | `true` still enables the Experiment Lab and logs a one-line deprecation at config load; the value is folded into `dashboard.experiment_lab` so the next config save writes the documented key. | | `dashboard.matrix` | Off by default (`enabled: false`, `repos: []`, `exclude_paths: []`). **Experimental.** | Code activity source; see [Code activity reader](../dashboard.md#code-activity-reader) for presentation, compatibility, capture and access policies. When active, pushed lines from allow-listed repos are served to authenticated read-role viewers via `GET /api/matrix/lines?since=&max=` (bounded backlog, `Cache-Control: no-store`; `404` when inactive) and `/api/events` emits a content-free `code-lines` nudge (`{"seq":N}`, ≤1/s) telling browsers to pull. `repos` is an explicit exact `owner/name` allowlist (no wildcards, URLs or bare names) and an empty list captures nothing, so `enabled: true` alone stays inactive. `exclude_paths` adds non-empty path globs to the built-in deny-list. There is no `authors` or `private_repos` key: authors are fixed and private repos are always excluded. | | `hub.contribute_announcement` | Empty/off by default. | Operator notice for contributors. Set from Governor → Hub or `/contribute` Management (owner/read-write): `text` is server-sanitised plain text capped at 500 characters and rendered only through text-safe sinks (empty clears), `level` is `info`/`warning`, `expires_at` is optional RFC3339, and the server rotates `id` whenever text changes so dismissed Operations/Profile banners reappear. Active notices show on `/contribute` Operations/Profile, above the Onboarding command block, and in connected relay terminals. | diff --git a/src/docs/settings.md b/src/docs/settings.md index ac4ddd5327..cdb826adb5 100644 --- a/src/docs/settings.md +++ b/src/docs/settings.md @@ -112,6 +112,12 @@ The **Hub settings** block at the bottom of the tab sets explicit task acceptanc The OpenAI-compatible gateways (LiteLLM, OpenRouter, vLLM, and others) agents send inference through, and their keys. See [Configure model gateways and keys](hosted-hub.md#configure-model-gateways-and-keys). +## Dashboard + +Settings for the dashboard itself (`dashboard.*`): its port and agent poll interval, the Experiment Lab, whether the header also shows the Claude usage gauge, hub-proxied auth, the dashboard auth token, the access allowlist, and the issue bands. + +**Also show the usage gauge in the header** (`dashboard.subscription_gauge_header`, off by default) adds a small fuel-dial chip to the header on every page, showing how much of the Claude plan this hive is signed in with is left in the limit closest to running out. The full gauge is on the Overview by default for the owner whenever the hive uses Claude, and can be hidden there from the Overview settings. Both show the measurements hive already makes, so they add no calls to Claude; only the owner sees them; and they show the Claude plan, not the hive's token budget. See [Claude usage gauge](dashboard.md#claude-usage-gauge). + ## Compliance How your configuration maps to SOC 2, FedRAMP, and ISO 27001 controls. See [The Settings → Compliance tab](compliance.md#the-settings--compliance-tab). diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 8b54b34f80..82df5431be 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -5290,6 +5290,13 @@ type DashboardConfig struct { // // Deprecated: set dashboard.experiment_lab instead. StrategyLab bool `yaml:"strategy_lab,omitempty" json:"strategy_lab,omitempty"` + // SubscriptionGaugeHeader also shows the owner the Claude subscription + // usage gauge as a chip in the dashboard header; the gauge is on the + // Overview by default and needs no config. It is display-only: it reads + // the usage readings hive already makes (provider rotation or the default + // publish-only manager) and never starts, stops or changes any + // measurement, rotation or agent. + SubscriptionGaugeHeader bool `yaml:"subscription_gauge_header,omitempty" json:"subscription_gauge_header,omitempty"` // Matrix is the default-off Code activity source configuration. // The dashboard.matrix name is deprecated but retained for compatibility. Matrix MatrixConfig `yaml:"matrix,omitempty" json:"matrix,omitempty"` diff --git a/src/pkg/dashboard/api_dashboard_config.go b/src/pkg/dashboard/api_dashboard_config.go index b6bcf1c4bf..7b7c98bd63 100644 --- a/src/pkg/dashboard/api_dashboard_config.go +++ b/src/pkg/dashboard/api_dashboard_config.go @@ -9,16 +9,17 @@ import ( ) type dashboardSettingsPayload struct { - Port int `json:"port"` - AgentPollIntervalS int `json:"agent_poll_interval_s"` - ExperimentLab bool `json:"experiment_lab"` - StrategyLab bool `json:"strategy_lab"` - AuthTokenConfigured bool `json:"auth_token_configured"` - AuthorizedUsers []string `json:"authorized_users"` - AuthorizedUserNames map[string]string `json:"authorized_user_names"` - HubProxied bool `json:"hub_proxied"` - IssueBands dashboardIssueBandsPayload `json:"issue_bands"` - StageTracker dashboardStageTrackerPayload `json:"stage_tracker"` + Port int `json:"port"` + AgentPollIntervalS int `json:"agent_poll_interval_s"` + ExperimentLab bool `json:"experiment_lab"` + StrategyLab bool `json:"strategy_lab"` + SubscriptionGaugeHeader bool `json:"subscription_gauge_header"` + AuthTokenConfigured bool `json:"auth_token_configured"` + AuthorizedUsers []string `json:"authorized_users"` + AuthorizedUserNames map[string]string `json:"authorized_user_names"` + HubProxied bool `json:"hub_proxied"` + IssueBands dashboardIssueBandsPayload `json:"issue_bands"` + StageTracker dashboardStageTrackerPayload `json:"stage_tracker"` } type dashboardIssueBandsPayload struct { @@ -39,14 +40,15 @@ func dashboardSettingsResponse(cfg *config.Config) dashboardSettingsPayload { toastInfo := int(cfg.StageTrackerToastInfo() / time.Second) toastError := int(cfg.StageTrackerToastError() / time.Second) return dashboardSettingsPayload{ - Port: d.Port, - AgentPollIntervalS: d.AgentPollIntervalS, - ExperimentLab: d.ExperimentLab, - StrategyLab: d.StrategyLab, - AuthTokenConfigured: strings.TrimSpace(d.AuthToken) != "", - AuthorizedUsers: append([]string{}, d.AuthorizedUsers...), - AuthorizedUserNames: names, - HubProxied: d.HubProxied, + Port: d.Port, + AgentPollIntervalS: d.AgentPollIntervalS, + ExperimentLab: d.ExperimentLab, + StrategyLab: d.StrategyLab, + SubscriptionGaugeHeader: d.SubscriptionGaugeHeader, + AuthTokenConfigured: strings.TrimSpace(d.AuthToken) != "", + AuthorizedUsers: append([]string{}, d.AuthorizedUsers...), + AuthorizedUserNames: names, + HubProxied: d.HubProxied, IssueBands: dashboardIssueBandsPayload{ WaitingLabels: append([]string{}, d.IssueBands.WaitingLabels...), DoneLabels: append([]string{}, d.IssueBands.DoneLabels...), @@ -68,16 +70,17 @@ func (s *Server) handleDashboardSettingsPut(w http.ResponseWriter, r *http.Reque return } var body struct { - Port *int `json:"port"` - AgentPollIntervalS *int `json:"agent_poll_interval_s"` - ExperimentLab *bool `json:"experiment_lab"` - StrategyLab *bool `json:"strategy_lab"` - AuthToken *string `json:"auth_token"` - AuthorizedUsers []string `json:"authorized_users"` - AuthorizedUserNames map[string]string `json:"authorized_user_names"` - HubProxied *bool `json:"hub_proxied"` - IssueBands *dashboardIssueBandsPayload `json:"issue_bands"` - StageTracker *dashboardStageTrackerPayload `json:"stage_tracker"` + Port *int `json:"port"` + AgentPollIntervalS *int `json:"agent_poll_interval_s"` + ExperimentLab *bool `json:"experiment_lab"` + StrategyLab *bool `json:"strategy_lab"` + SubscriptionGaugeHeader *bool `json:"subscription_gauge_header"` + AuthToken *string `json:"auth_token"` + AuthorizedUsers []string `json:"authorized_users"` + AuthorizedUserNames map[string]string `json:"authorized_user_names"` + HubProxied *bool `json:"hub_proxied"` + IssueBands *dashboardIssueBandsPayload `json:"issue_bands"` + StageTracker *dashboardStageTrackerPayload `json:"stage_tracker"` } if err := decodeBody(r, &body); err != nil { jsonError(w, "invalid body", http.StatusBadRequest) @@ -109,6 +112,9 @@ func (s *Server) handleDashboardSettingsPut(w http.ResponseWriter, r *http.Reque if body.StrategyLab != nil { next.StrategyLab = *body.StrategyLab } + if body.SubscriptionGaugeHeader != nil { + next.SubscriptionGaugeHeader = *body.SubscriptionGaugeHeader + } if body.AuthToken != nil { next.AuthToken = strings.TrimSpace(*body.AuthToken) } diff --git a/src/pkg/dashboard/api_dashboard_config_test.go b/src/pkg/dashboard/api_dashboard_config_test.go index 7b503fd120..81e12ec150 100644 --- a/src/pkg/dashboard/api_dashboard_config_test.go +++ b/src/pkg/dashboard/api_dashboard_config_test.go @@ -1,10 +1,15 @@ package dashboard import ( + "bytes" "encoding/json" "net/http" + "net/http/httptest" "strings" "testing" + + "github.com/hivecommons/hive/pkg/config" + "gopkg.in/yaml.v3" ) func TestDashboardSettingsAPIRoundTripAndMasksToken(t *testing.T) { @@ -63,3 +68,92 @@ func TestDashboardSettingsAPIRejectsInvalidNumbers(t *testing.T) { t.Fatalf("invalid PUT mutated dashboard.port to %d", s.deps.Config.Dashboard.Port) } } + +// TestDashboardSettingsAPISubscriptionGaugeHeaderToggle covers the display-only +// dashboard.subscription_gauge_header flag: off by default, owner can turn it on and +// off, and an omitted field leaves it unchanged. +func TestDashboardSettingsAPISubscriptionGaugeHeaderToggle(t *testing.T) { + s := govServer(t) + + get := doOwnerGet(s, "/api/config/dashboard") + if get.Code != http.StatusOK { + t.Fatalf("GET = %d: %s", get.Code, get.Body.String()) + } + var got dashboardSettingsPayload + if err := json.Unmarshal(get.Body.Bytes(), &got); err != nil { + t.Fatalf("GET body: %v", err) + } + if got.SubscriptionGaugeHeader { + t.Fatal("subscription_gauge_header = true by default, want false") + } + + put := doPut(s, "/api/config/dashboard", map[string]any{"subscription_gauge_header": true}) + if put.Code != http.StatusOK { + t.Fatalf("PUT on = %d: %s", put.Code, put.Body.String()) + } + if !s.deps.Config.Dashboard.SubscriptionGaugeHeader { + t.Fatal("dashboard.subscription_gauge_header not saved as true") + } + if err := json.Unmarshal(put.Body.Bytes(), &got); err != nil || !got.SubscriptionGaugeHeader { + t.Fatalf("PUT response subscription_gauge_header = %v (err %v), want true", got.SubscriptionGaugeHeader, err) + } + + if rec := doPut(s, "/api/config/dashboard", map[string]any{"agent_poll_interval_s": 9}); rec.Code != http.StatusOK { + t.Fatalf("unrelated PUT = %d: %s", rec.Code, rec.Body.String()) + } + if !s.deps.Config.Dashboard.SubscriptionGaugeHeader { + t.Fatal("a PUT without subscription_gauge_header turned the flag off") + } + + if rec := doPut(s, "/api/config/dashboard", map[string]any{"subscription_gauge_header": false}); rec.Code != http.StatusOK { + t.Fatalf("PUT off = %d: %s", rec.Code, rec.Body.String()) + } + if s.deps.Config.Dashboard.SubscriptionGaugeHeader { + t.Fatal("dashboard.subscription_gauge_header not saved as false") + } +} + +func TestDashboardSettingsAPISubscriptionGaugeHeaderRejectsNonOwner(t *testing.T) { + s := govServer(t) + var b bytes.Buffer + _ = json.NewEncoder(&b).Encode(map[string]any{"subscription_gauge_header": true}) + req := httptest.NewRequest(http.MethodPut, "/api/config/dashboard", &b) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Hive-Role", "read-write") + rec := httptest.NewRecorder() + + s.mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("non-owner PUT status = %d, want 403", rec.Code) + } + if s.deps.Config.Dashboard.SubscriptionGaugeHeader { + t.Fatal("non-owner PUT changed dashboard.subscription_gauge_header") + } +} + +// TestDashboardSubscriptionGaugeHeaderOmittedFromYAMLWhenOff checks that a hive +// that never turns the gauge on keeps a saved config without the key. +func TestDashboardSubscriptionGaugeHeaderOmittedFromYAMLWhenOff(t *testing.T) { + out, err := yaml.Marshal(config.DashboardConfig{}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "subscription_gauge_header") { + t.Fatalf("default dashboard config emitted subscription_gauge_header:\n%s", out) + } + out, err = yaml.Marshal(config.DashboardConfig{SubscriptionGaugeHeader: true}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "subscription_gauge_header: true") { + t.Fatalf("enabled dashboard config lacks subscription_gauge_header: true:\n%s", out) + } + var cfg config.Config + if err := yaml.Unmarshal([]byte("dashboard:\n subscription_gauge_header: true\n"), &cfg); err != nil { + t.Fatal(err) + } + if !cfg.Dashboard.SubscriptionGaugeHeader { + t.Fatal("dashboard.subscription_gauge_header: true was not read") + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index a650064ffc..1c61c1dc97 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -771,6 +771,10 @@ type FrontendFeatures struct { // // Deprecated: read experiment_lab instead. StrategyLab bool `json:"strategy_lab"` + // SubscriptionGaugeHeader adds the owner-only Claude subscription usage + // chip to the header (the Overview gauge needs no flag). Only the flag + // travels here; owners fetch readings from /api/providers/headroom. + SubscriptionGaugeHeader bool `json:"subscription_gauge_header"` } // FrontendSecurity summarizes the effective operator security posture for compact dashboard display. diff --git a/src/pkg/dashboard/static-src/css/overview.css b/src/pkg/dashboard/static-src/css/overview.css index 07ad574b6c..3ff5e3a455 100644 --- a/src/pkg/dashboard/static-src/css/overview.css +++ b/src/pkg/dashboard/static-src/css/overview.css @@ -171,3 +171,37 @@ .overview-pr-open { --slice-c: var(--band-open); } .overview-pr-draft { --slice-c: var(--band-draft); } + /* Claude subscription usage gauge: Overview block and fuel dials. */ + .overview-usage-gauge { display: flex; flex-direction: column; gap: var(--sp-3); margin: var(--sp-0) var(--sp-0) var(--sp-4); padding: var(--sp-4); border: 1px solid var(--border); border-radius: var(--r-lg); background: var(--surface-2); } + .overview-usage-gauge[hidden] { display: none; } + .overview-usage-gauge__head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: var(--sp-3); } + .overview-usage-gauge__title { font-size: var(--fs-md); font-weight: var(--fw-bold); } + .overview-usage-gauge__meta, .overview-usage-gauge__reset, .overview-usage-gauge__note { font-size: var(--fs-sm); color: var(--text-muted); } + .overview-usage-gauge__stale { font-size: var(--fs-sm); color: var(--status-warn); } + .overview-usage-gauge__dials { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: var(--sp-4); } + .overview-usage-gauge__dial { display: flex; flex-direction: column; align-items: center; gap: var(--sp-1); text-align: center; } + .overview-usage-gauge__label { font-weight: var(--fw-semibold); } + .overview-usage-gauge__pct { font-size: var(--fs-lg); font-weight: var(--fw-bold); color: var(--status-ok); } + .overview-usage-gauge__pct[data-status="warn"] { color: var(--status-warn); } + .overview-usage-gauge__pct[data-status="error"] { color: var(--status-error); } + .overview-usage-gauge__pct[data-status="neutral"] { color: var(--text-muted); } + .sg-dial { display: block; width: 150px; height: auto; overflow: visible; } + .sg-dial--compact { width: 34px; } + .sg-dial__track, .sg-dial__zone, .sg-dial__fill { fill: none; stroke-width: 7; } + .sg-dial--compact .sg-dial__track, .sg-dial--compact .sg-dial__zone, .sg-dial--compact .sg-dial__fill { stroke-width: 12; } + .sg-dial__track { stroke: var(--border); } + .sg-dial__zone--red { stroke: var(--status-error); stroke-opacity: 0.55; } + .sg-dial__zone--amber { stroke: var(--status-warn); stroke-opacity: 0.55; } + .sg-dial__fill { stroke: var(--status-ok); stroke-opacity: 0.35; } + .sg-dial__tick { fill: none; stroke: var(--text-muted); stroke-width: 1.6; } + .sg-dial__letter { fill: var(--text-muted); font-weight: var(--fw-bold); } + .sg-dial__letter--e { fill: var(--status-error); } + .sg-dial__needle { stroke: var(--status-ok); stroke-width: 3.5; stroke-linecap: round; } + .sg-dial--compact .sg-dial__needle { stroke-width: 7; } + .sg-dial__hub { fill: var(--status-ok); } + .sg-dial[data-status="warn"] .sg-dial__needle, .sg-dial[data-status="warn"] .sg-dial__fill { stroke: var(--status-warn); } + .sg-dial[data-status="warn"] .sg-dial__hub { fill: var(--status-warn); } + .sg-dial[data-status="error"] .sg-dial__needle, .sg-dial[data-status="error"] .sg-dial__fill { stroke: var(--status-error); } + .sg-dial[data-status="error"] .sg-dial__hub { fill: var(--status-error); } + .sg-dial[data-stale="true"] .sg-dial__needle, .sg-dial[data-stale="true"] .sg-dial__fill { stroke: var(--text-muted); } + .sg-dial[data-stale="true"] .sg-dial__hub { fill: var(--text-muted); } diff --git a/src/pkg/dashboard/static-src/css/topbar.css b/src/pkg/dashboard/static-src/css/topbar.css index 93334c2f74..aa5b071b62 100644 --- a/src/pkg/dashboard/static-src/css/topbar.css +++ b/src/pkg/dashboard/static-src/css/topbar.css @@ -90,6 +90,11 @@ `[hidden] { display: none }`, so a hidden status chip otherwise paints as an empty outlined pill (#oc-providers when every provider is usable). */ .oc-providers-chip[hidden] { display: none; } + /* Opt-in Claude usage chip: a compact fuel dial plus its percentage. */ + .subscription-gauge-chip[hidden] { display: none; } + .subscription-gauge-chip__dial { display: inline-flex; align-items: center; } + .subscription-gauge-chip__dial:empty { display: none; } + .subscription-gauge-chip[data-out-of-date="true"] { border-style: dashed; } .oc-version-navbar-upgrade { --oc-version-bee-lane: 56px; position: relative; display: flex; align-items: center; min-width: 0; max-width: 100%; flex: 1 1 auto; overflow: visible; } .oc-version-navbar-upgrade .oc-version-hive { position: absolute; left: 0; top: 50%; transform: translateY(-50%); } .oc-version-navbar-upgrade .oc-version-honey-dot { width: 4px; height: 4px; } @@ -216,6 +221,7 @@ #acmm-badge { display: none; } .oc-health-badge .nav-chip__label, #oc-settings-btn .nav-chip__label, + #oc-subscription-gauge .nav-chip__label, #welcome-topbar-btn .nav-chip__label { display: none; } } diff --git a/src/pkg/dashboard/static-src/html/overview-governor.html b/src/pkg/dashboard/static-src/html/overview-governor.html index 97e48ef0cc..1bc29e40a5 100644 --- a/src/pkg/dashboard/static-src/html/overview-governor.html +++ b/src/pkg/dashboard/static-src/html/overview-governor.html @@ -10,6 +10,7 @@ <h2 class="config-title" id="overview-chart-settings-title">Overview settings</h </div> </div> <div class="section-body"> + <div class="overview-usage-gauge" id="overview-usage-gauge" role="region" aria-label="Claude subscription usage" hidden></div> <div class="overview-card" id="overview-card"></div> </div> </div> diff --git a/src/pkg/dashboard/static-src/html/topbar.html b/src/pkg/dashboard/static-src/html/topbar.html index 4ceaff3d88..fa571b8ed3 100644 --- a/src/pkg/dashboard/static-src/html/topbar.html +++ b/src/pkg/dashboard/static-src/html/topbar.html @@ -31,6 +31,7 @@ <!-- Agent Logs hidden (redundant with per-agent output) --> <button type="button" id="oc-navbar-upgrade" class="nav-chip nav-chip--status nav-chip--warn oc-navbar-upgrade" data-action="focusVersionBlock" title="Upgrade progress — jump to Version details" aria-label="Upgrade progress" hidden></button> <span class="nav-chip nav-chip--status oc-providers-chip" id="oc-providers" data-status="warn" role="status" title="AI providers: all usable" aria-label="AI providers: all usable" hidden></span> + <button type="button" class="nav-chip nav-chip--status subscription-gauge-chip" id="oc-subscription-gauge" data-action="openSubscriptionGaugeOverview" data-status="neutral" aria-label="Claude subscription usage left" hidden><span class="subscription-gauge-chip__dial" aria-hidden="true"></span><span class="nav-chip__label"></span></button> <button type="button" class="nav-chip nav-chip--status nav-chip--attention oc-waiting-on-you" id="oc-waiting-on-you" data-action="showWaitingOnYou" title="Nothing is waiting on you" aria-label="0 waiting on you — show in the activity monitor" hidden><span class="nav-chip__dot" aria-hidden="true">●</span> <span class="nav-chip__label">Waiting on you · 0</span></button> <span class="nav-chip nav-chip--status nav-chip--ok oc-health-badge" id="oc-health"><span class="nav-chip__dot" aria-hidden="true">●</span> <span class="nav-chip__label">Health OK</span></span> <button class="nav-chip nav-chip--primary" id="welcome-topbar-btn" style="display:none" data-action="openWelcomeDialog" title="Open the Getting Started guide" aria-label="Guide"><span class="nav-chip__icon" aria-hidden="true">🧭</span><span class="nav-chip__label">Guide</span></button> diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index c3b76f8bfa..b643f0e4a6 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -90,6 +90,7 @@ { tab: 'Features', section: 'Work intake & issue lifecycle', label: 'Stall replan', key: 'governor.replan.enabled', text: 'stall replan approved plans architect interval threshold max replans' }, { tab: 'Advisory', section: 'Queue health', label: 'Queue health thresholds', key: 'governor.advisory.queue_health.blocked_pr_pct', text: 'advisory queue health blocked needs human stale lane share check share thresholds' }, { tab: 'Dashboard', section: 'Dashboard runtime', label: 'Experiment Lab', key: 'dashboard.experiment_lab', text: 'experiment_lab Experiment Lab dashboard feature experimental planner' }, + { tab: 'Dashboard', section: 'Dashboard runtime', label: 'Subscription usage gauge in header', key: 'dashboard.subscription_gauge_header', text: 'subscription_gauge_header subscription usage gauge fuel header chip claude usage claude plan weekly 5-hour limit left remaining quota headroom' }, { tab: 'Project Observability', section: 'Target platforms', label: 'Project observability targets', key: 'governor.project_observability.open_source[]', text: 'project observability telemetry operations open source kube native commercial references endpoint env credential secret' }, { tab: 'Features', section: 'Work intake & issue lifecycle', label: 'Question auto-close', key: 'governor.question_autoclose.enabled', text: 'question auto close answered questions labels human objection window' }, { tab: 'Logging', section: 'Logging', label: 'Kick log source', key: 'logging.kick_log_source', text: 'kick log source live file audit' }, diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index fd02e566d9..bdfdb07771 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -886,6 +886,7 @@ '<div class="config-field"><label>Agent poll interval (seconds) ' + yamlHint('dashboard.agent_poll_interval_s') + '</label><input type="number" min="1" value="' + esc(String(dash.agent_poll_interval_s || 0)) + '" data-change-action="markDirty" data-arg0="dashboard" data-arg1="agent_poll_interval_s" data-arg-types="s,s,N"></div>' + '</div>' + '<div class="config-toggle"><div class="config-toggle-switch ' + (dash.experiment_lab ? 'on' : '') + '" data-action="toggleConfigSwitch" data-section="dashboard" data-key="experiment_lab"></div><span class="config-toggle-label">Experiment Lab ' + yamlHint('dashboard.experiment_lab') + '</span></div>' + + '<div class="config-toggle"><div class="config-toggle-switch ' + (dash.subscription_gauge_header ? 'on' : '') + '" data-action="toggleConfigSwitch" data-section="dashboard" data-key="subscription_gauge_header"></div><span class="config-toggle-label">Also show the usage gauge in the header ' + yamlHint('dashboard.subscription_gauge_header') + ' ' + settingHelpMark('settings', 'dashboard', 'Usage gauge in the header', 'The Claude usage fuel gauge is on the Overview by default for the owner. This also shows it as a small chip in the header on every page. It reads the usage measurements hive already makes, so it adds no calls to Claude. Only the hive owner sees it. This is the Claude plan this hive is signed in with, not the hive token budget. Off by default.') + '</span></div>' + '<div class="config-toggle"><div class="config-toggle-switch ' + (dash.hub_proxied ? 'on' : '') + '" data-action="toggleConfigSwitch" data-section="dashboard" data-key="hub_proxied"></div><span class="config-toggle-label">Hub-proxied auth ' + yamlHint('dashboard.hub_proxied') + ' <span class="text-muted">Managed by hub on hosted hives.</span></span></div>' + '<div class="config-field"><label>Dashboard auth token ' + yamlHint('dashboard.auth_token') + '</label><input type="password" autocomplete="new-password" placeholder="' + esc(authTokenText) + '" data-change-action="markDirty" data-arg0="dashboard" data-arg1="auth_token" data-arg-types="s,s,v"><div class="config-help">Write-only; the current token is never echoed.</div></div>' + '<h3>Access allowlist</h3>' + diff --git a/src/pkg/dashboard/static-src/js/gh-auth.js b/src/pkg/dashboard/static-src/js/gh-auth.js index 35e4c6bf6c..4a9329da93 100644 --- a/src/pkg/dashboard/static-src/js/gh-auth.js +++ b/src/pkg/dashboard/static-src/js/gh-auth.js @@ -66,6 +66,7 @@ const repaintTracker = hiveAuthResolved(); if (username) window._hiveUser = username; if (role) window._hiveRole = role; + if (typeof subscriptionGaugeRefresh === 'function') subscriptionGaugeRefresh(); const banner = document.getElementById('gh-auth-banner'); const topBtn = document.getElementById('oc-gh-login-btn'); const avatarWrap = document.getElementById('oc-gh-avatar-wrap'); diff --git a/src/pkg/dashboard/static-src/js/overview.js b/src/pkg/dashboard/static-src/js/overview.js index c2235514a6..3395bf904c 100644 --- a/src/pkg/dashboard/static-src/js/overview.js +++ b/src/pkg/dashboard/static-src/js/overview.js @@ -501,7 +501,7 @@ function overviewDefaultChartState() { const enabled = {}; OVERVIEW_CHART_TYPES.forEach(type => { enabled[type] = type === 'donut'; }); - return { enabled, carousel: false, intervalMs: OVERVIEW_CAROUSEL_DEFAULT_MS, transition: 'fade', duration: 'normal', active: { issue: 'donut', pr: 'donut' }, history: { issue: [], pr: [] }, labelMode: 'both', showKPIs: true, exportFormat: 'csv' }; + return { enabled, carousel: false, intervalMs: OVERVIEW_CAROUSEL_DEFAULT_MS, transition: 'fade', duration: 'normal', active: { issue: 'donut', pr: 'donut' }, history: { issue: [], pr: [] }, labelMode: 'both', showKPIs: true, showUsageGauge: true, exportFormat: 'csv' }; } function overviewChartsState() { @@ -518,7 +518,7 @@ const history = saved.history && typeof saved.history === 'object' ? saved.history : base.history; const labelMode = OVERVIEW_LABEL_MODES.includes(saved.labelMode) ? saved.labelMode : base.labelMode; const exportFormat = OVERVIEW_EXPORT_FORMATS.includes(saved.exportFormat) ? saved.exportFormat : base.exportFormat; - return { enabled, carousel: !!saved.carousel, intervalMs, transition, duration, active, history, labelMode, showKPIs: saved.showKPIs !== false, exportFormat }; + return { enabled, carousel: !!saved.carousel, intervalMs, transition, duration, active, history, labelMode, showKPIs: saved.showKPIs !== false, showUsageGauge: saved.showUsageGauge !== false, exportFormat }; } const OVERVIEW_KPI_WINDOWS = { '24h': 24 * 3600e3, '7d': 7 * 24 * 3600e3 }; @@ -1414,7 +1414,7 @@ const autoTitle = state.carousel ? 'Auto-play on — rotates every ' + Math.round(overviewCarouselDwellMs(state) / 1000) + 's through the ticked charts. Click to pause.' : 'Auto-play off — click to rotate through the ticked charts every ' + Math.round(overviewCarouselDwellMs(state) / 1000) + 's'; return `<div class="overview-settings-section" aria-label="${esc(label)} chart controls"><div class="overview-settings-card-title">${esc(label)}</div><div class="overview-chart-controls"><button type="button" class="overview-carousel-arrow" data-action="overviewCarouselPrev" data-arg0="${kind}" data-stop="1" title="Previous ${esc(label)} chart" aria-label="Previous ${esc(label)} chart">◀</button><label class="overview-settings-field">Chart type<select class="overview-chart-select" data-change-action="setOverviewChartType" data-arg0="${kind}" data-arg-types="s,v" title="${esc(label)} chart type">${overviewChartTypeOptions(active)}</select></label><button type="button" class="overview-carousel-arrow" data-action="overviewCarouselNext" data-arg0="${kind}" data-stop="1" title="Next ${esc(label)} chart" aria-label="Next ${esc(label)} chart">▶</button>${renderOverviewDots(enabled, active)}<span class="overview-chart-controls-sep"></span><button type="button" class="overview-carousel-arrow" data-action="toggleOverviewCarousel" data-stop="1" aria-pressed="${state.carousel ? 'true' : 'false'}" title="${esc(autoTitle)}" aria-label="${esc(autoTitle)}">${state.carousel ? '⏸ auto' : '⏵ auto'}</button><label class="overview-settings-field">Transition<select class="overview-chart-select" data-change-action="setOverviewTransition" data-arg-types="v" title="Transition between charts">${overviewTransitionOptions(state.transition)}</select></label></div></div>`; }; - setIfChanged(el, `<div class="overview-settings-section"><div class="overview-settings-card-title">Overview charts</div><div class="overview-settings-grid">${typeRows}</div><div class="overview-settings-grid"><label class="overview-settings-field">Interval (seconds)<input type="number" min="5" max="300" step="5" value="${seconds}" data-change-action="setOverviewCarouselInterval" data-arg-types="N"></label><label class="overview-settings-field">Duration<select data-change-action="setOverviewDuration" data-arg-types="v">${durationOptions}</select></label><label class="overview-settings-field">Export format<select data-change-action="setOverviewExportFormat" data-arg-types="v">${exportOptions}</select></label><label class="overview-settings-field">Donut labels<select data-change-action="setOverviewLabelMode" data-arg-types="v">${labelOptions}</select></label><label class="overview-settings-row"><input type="checkbox" data-change-action="setOverviewKPIs" data-arg-types="c"${state.showKPIs ? ' checked' : ''}>Show KPI strip</label></div><div class="overview-settings-links"><button type="button" class="hv-btn btn-sm" data-action="resetOverviewSettings" data-stop="1">Reset to defaults</button></div></div>${overviewRepoFilterHTML(_overviewLastRepos || [])}${cardControls('issue', 'Issues')}${cardControls('pr', 'PRs')}`); + setIfChanged(el, `<div class="overview-settings-section"><div class="overview-settings-card-title">Overview charts</div><div class="overview-settings-grid">${typeRows}</div><div class="overview-settings-grid"><label class="overview-settings-field">Interval (seconds)<input type="number" min="5" max="300" step="5" value="${seconds}" data-change-action="setOverviewCarouselInterval" data-arg-types="N"></label><label class="overview-settings-field">Duration<select data-change-action="setOverviewDuration" data-arg-types="v">${durationOptions}</select></label><label class="overview-settings-field">Export format<select data-change-action="setOverviewExportFormat" data-arg-types="v">${exportOptions}</select></label><label class="overview-settings-field">Donut labels<select data-change-action="setOverviewLabelMode" data-arg-types="v">${labelOptions}</select></label><label class="overview-settings-row"><input type="checkbox" data-change-action="setOverviewKPIs" data-arg-types="c"${state.showKPIs ? ' checked' : ''}>Show KPI strip</label>${typeof subscriptionGaugeOwner === 'function' && subscriptionGaugeOwner() ? `<label class="overview-settings-row"><input type="checkbox" data-change-action="setOverviewUsageGauge" data-arg-types="c"${state.showUsageGauge ? ' checked' : ''}>Show Claude usage gauge</label>` : ''}</div><div class="overview-settings-links"><button type="button" class="hv-btn btn-sm" data-action="resetOverviewSettings" data-stop="1">Reset to defaults</button></div></div>${overviewRepoFilterHTML(_overviewLastRepos || [])}${cardControls('issue', 'Issues')}${cardControls('pr', 'PRs')}`); } function rerenderOverviewChartsFromState(state) { @@ -1486,6 +1486,13 @@ rerenderOverviewChartsFromState(state); } + function setOverviewUsageGauge(checked) { + const state = overviewChartsState(); + state.showUsageGauge = !!checked; + rerenderOverviewChartsFromState(state); + if (typeof subscriptionGaugeRefresh === 'function') subscriptionGaugeRefresh(); + } + function setOverviewRepoFilter(repo, checked) { const all = overviewAllRepoNames(_overviewLastRepos || []); const selected = new Set(overviewSelectedRepoNames(_overviewLastRepos || [])); diff --git a/src/pkg/dashboard/static-src/js/refresh.js b/src/pkg/dashboard/static-src/js/refresh.js index 8e4e569066..4efb331c8d 100644 --- a/src/pkg/dashboard/static-src/js/refresh.js +++ b/src/pkg/dashboard/static-src/js/refresh.js @@ -528,6 +528,7 @@ // Update Light health badge with hover detail renderNavbarHealthBadge(document.getElementById('oc-health'), data); renderProviderStrip(document.getElementById('oc-providers'), data.providers); + if (typeof subscriptionGaugeRefresh === 'function') subscriptionGaugeRefresh(); renderGhRateLimits(data.ghRateLimits || {}); renderGitHubAppBanner(data); renderRepoTargetBanner(data); diff --git a/src/pkg/dashboard/static-src/js/subscription-gauge.js b/src/pkg/dashboard/static-src/js/subscription-gauge.js new file mode 100644 index 0000000000..3371b9f285 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/subscription-gauge.js @@ -0,0 +1,300 @@ + // Claude subscription usage gauge. subscriptionGaugeModel is the single + // pure decision point: it turns a /api/providers/headroom response into + // the dials, the needle and the no-reading reason that the Overview gauge + // and the header chip render. No DOM, no fetch, so node tests can run it. + function subscriptionGaugeModel(resp, nowMs) { + const OUT_OF_DATE_MS = 15 * 60 * 1000; + const REASONS = { + not_installed: 'Claude Code is not installed on this hive.', + no_credentials: 'This hive is not signed in to Claude.', + rate_limited: 'Claude rate-limited the usage check; hive will try again.', + timeout: 'The usage check timed out; hive will try again.', + unrecognized_schema: 'Claude changed its usage format, so hive could not read it.', + probe_failed: 'The usage check failed; hive will try again.', + }; + const NOT_USED = ['not_installed', 'no_credentials']; + const validTime = (v) => { + const ms = Date.parse(v || ''); + // Go's zero time.Time serializes as 0001-01-01: treat it as unknown. + return Number.isFinite(ms) && ms > 0 ? ms : null; + }; + const statusFor = (pct) => (pct <= 10 ? 'error' : pct <= 25 ? 'warn' : 'ok'); + const describe = (kind, id, mins, scope) => { + const k = String(kind || id || '').toLowerCase(); + const i = String(id || '').toLowerCase(); + const sc = scope || {}; + // A weekly limit that counts only some models: the builtin source + // sends kind weekly_scoped; ccleft sends kind weekly with id + // seven_day_<model> and scope.scope set (binding "false"). + const scoped = k === 'weekly_scoped' || ((k === 'weekly' || k === 'seven_day') && + (!!sc.scope || /^(seven_day|weekly)_(?!all$)./.test(i))); + if (scoped) { + let model = sc.model || sc.model_family || sc.scope || sc.name || ''; + if (/^scoped$/i.test(String(model))) model = ''; + return { label: model ? 'Weekly (' + model + ')' : 'Weekly (model-specific)', shortName: 'model' }; + } + if (k === 'weekly' || k === 'weekly_all' || k === 'seven_day') return { label: 'Weekly', shortName: '' }; + if (k === 'session' || k === 'five_hour') return { label: '5-hour', shortName: '5-hour' }; + if (k === 'daily') return { label: 'Daily', shortName: 'daily' }; + if (k === 'monthly') return { label: 'Monthly', shortName: 'monthly' }; + if (mins > 0 && mins % 1440 === 0) return { label: (mins / 1440) + '-day', shortName: (mins / 1440) + '-day' }; + if (mins > 0) return { label: Math.round(mins / 60) + '-hour', shortName: Math.round(mins / 60) + '-hour' }; + return { label: 'Limit', shortName: 'limit' }; + }; + const providers = resp && Array.isArray(resp.providers) ? resp.providers : []; + const reading = providers.find((p) => p && p.provider === 'anthropic'); + if (!reading) { + return { state: 'no-reading', usesClaude: false, cause: 'absent', reason: 'Usage measurement is not running on this hive.' }; + } + const cause = String(reading.probe_error_cause || ''); + const limits = Array.isArray(reading.limits) ? reading.limits : []; + const dials = []; + // Paid credit pools and prepaid balances are not plan limits. + const NOT_LIMITS = ['credits', 'balance']; + limits.forEach((l, i) => { + if (!l || typeof l.pct_remaining !== 'number' || !Number.isFinite(l.pct_remaining)) return; + if (NOT_LIMITS.includes(String(l.kind || '').toLowerCase())) return; + const mins = Number(l.duration_mins) || 0; + const d = describe(l.kind, l.id, mins, l.scope); + const pctLeft = Math.max(0, Math.min(100, Math.round(l.pct_remaining))); + const reset = validTime(l.resets_at); + dials.push({ + id: String(l.id || l.kind || ''), + kind: String(l.kind || ''), + label: d.label, + shortName: d.shortName, + pctLeft: pctLeft, + resetsAt: reset === null ? '' : new Date(reset).toISOString(), + durationMins: mins, + status: statusFor(pctLeft), + order: i, + }); + }); + if (!dials.length) { + return { + state: 'no-reading', + usesClaude: !NOT_USED.includes(cause), + cause: cause || 'empty', + reason: REASONS[cause] || 'No usage reading yet.', + }; + } + // Longest window first; among equal lengths the all-models weekly + // limit leads, then the provider's own order. + const rank = (dial) => (dial.shortName === '' ? 0 : 1); + dials.sort((a, b) => { + const am = a.durationMins > 0 ? a.durationMins : -1; + const bm = b.durationMins > 0 ? b.durationMins : -1; + if (am !== bm) return bm - am; + if (rank(a) !== rank(b)) return rank(a) - rank(b); + return a.order - b.order; + }); + // The needle follows the window with the least left; ties keep the + // earlier (longer) window. + let needle = dials[0]; + dials.forEach((dial) => { if (dial.pctLeft < needle.pctLeft) needle = dial; }); + const captured = validTime(reading.captured_at); + const ageMs = captured === null || !Number.isFinite(nowMs) ? null : Math.max(0, nowMs - captured); + return { + state: 'reading', + usesClaude: true, + cause: cause, + dials: dials.map((dial) => ({ + id: dial.id, kind: dial.kind, label: dial.label, shortName: dial.shortName, + pctLeft: dial.pctLeft, resetsAt: dial.resetsAt, durationMins: dial.durationMins, status: dial.status, + })), + needle: { id: needle.id, kind: needle.kind, label: needle.label, shortName: needle.shortName, pctLeft: needle.pctLeft, status: needle.status }, + worstStatus: needle.status, + capturedAt: captured === null ? '' : new Date(captured).toISOString(), + ageMs: ageMs, + outOfDate: !!reading.stale || (ageMs !== null && ageMs > OUT_OF_DATE_MS), + planName: String(reading.plan_type || ''), + accountNote: 'Reflects the Claude account this hive is signed in with.', + }; + } + + // Shared loader. The headroom endpoint is owner-only and the fetch + // wrapper drops a stored token on any 401/403 from /api/, so nothing is + // fetched until the viewer is known to be the owner. The endpoint serves + // stored readings: polling it never triggers a measurement. + let _subscriptionGaugeData = null; + const SUBSCRIPTION_GAUGE_POLL_MS = 60000; + function subscriptionGaugeOwner() { + return typeof dashboardRoleAtLeast === 'function' && dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + } + function subscriptionGaugeOverviewEnabled() { + try { return typeof overviewChartsState !== 'function' || overviewChartsState().showUsageGauge !== false; } catch (e) { return true; } + } + function subscriptionGaugeHeaderEnabled() { + return typeof dashboardFeatureEnabled === 'function' && dashboardFeatureEnabled('subscription_gauge_header'); + } + function subscriptionGaugeWanted() { + return subscriptionGaugeOwner() && (subscriptionGaugeOverviewEnabled() || subscriptionGaugeHeaderEnabled()); + } + function subscriptionGaugeCurrentModel() { + return _subscriptionGaugeData ? subscriptionGaugeModel(_subscriptionGaugeData, Date.now()) : null; + } + function subscriptionGaugeRefresh() { + if (!subscriptionGaugeWanted()) { + dashboardClearInterval('subscription-gauge'); + } else if (!DASHBOARD_INTERVALS['subscription-gauge']) { + dashboardSetInterval('subscription-gauge', subscriptionGaugeFetch, SUBSCRIPTION_GAUGE_POLL_MS); + subscriptionGaugeFetch(); + } + renderSubscriptionGaugeSurfaces(); + } + async function subscriptionGaugeFetch() { + if (!subscriptionGaugeWanted()) return; + try { + const res = await fetch('/api/providers/headroom'); + if (!res.ok) return; + _subscriptionGaugeData = await res.json(); + } catch (e) { + return; + } + renderSubscriptionGaugeSurfaces(); + } + function renderSubscriptionGaugeSurfaces() { + renderSubscriptionGaugeOverview(); + if (typeof renderSubscriptionGaugeChip === 'function') renderSubscriptionGaugeChip(); + } + + // An E-to-F fuel dial drawn with DOM calls: numbers only, colours from + // CSS classes keyed on data-status, no inline styles. + function subscriptionGaugeDialSvg(pctLeft, opts) { + const o = opts || {}; + const NS = 'http://www.w3.org/2000/svg'; + const cx = 50, cy = 54, r = 42; + const v = Math.max(0, Math.min(100, Number(pctLeft) || 0)); + const pt = (f, rr) => { + const a = Math.PI * (1 - f); + return (cx + rr * Math.cos(a)).toFixed(2) + ' ' + (cy - rr * Math.sin(a)).toFixed(2); + }; + const arc = (a, b) => 'M ' + pt(a, r) + ' A ' + r + ' ' + r + ' 0 0 1 ' + pt(b, r); + const node = (tag, attrs) => { + const el = document.createElementNS(NS, tag); + Object.keys(attrs).forEach((k) => el.setAttribute(k, String(attrs[k]))); + return el; + }; + const svg = node('svg', { + viewBox: '0 0 100 62', + class: 'sg-dial' + (o.compact ? ' sg-dial--compact' : ''), + 'data-status': o.status || (v <= 10 ? 'error' : v <= 25 ? 'warn' : 'ok'), + 'data-stale': o.stale ? 'true' : 'false', + role: 'img', + 'aria-label': o.label || (v + '% left'), + }); + svg.appendChild(node('path', { class: 'sg-dial__track', d: arc(0, 1) })); + svg.appendChild(node('path', { class: 'sg-dial__zone sg-dial__zone--red', d: arc(0, 0.10) })); + svg.appendChild(node('path', { class: 'sg-dial__zone sg-dial__zone--amber', d: arc(0.10, 0.25) })); + if (v > 0.5) svg.appendChild(node('path', { class: 'sg-dial__fill', d: arc(0, v / 100) })); + if (!o.compact) { + let ticks = ''; + [0, 0.25, 0.5, 0.75, 1].forEach((f) => { ticks += 'M ' + pt(f, r - 9) + ' L ' + pt(f, r - 14) + ' '; }); + svg.appendChild(node('path', { class: 'sg-dial__tick', d: ticks.trim() })); + const e = node('text', { class: 'sg-dial__letter sg-dial__letter--e', x: 6, y: 60, 'font-size': 11 }); + e.textContent = 'E'; + const f = node('text', { class: 'sg-dial__letter', x: 87, y: 60, 'font-size': 11 }); + f.textContent = 'F'; + svg.appendChild(e); + svg.appendChild(f); + } + const tip = pt(v / 100, r - 5).split(' '); + svg.appendChild(node('line', { class: 'sg-dial__needle', x1: cx, y1: cy, x2: tip[0], y2: tip[1] })); + svg.appendChild(node('circle', { class: 'sg-dial__hub', cx: cx, cy: cy, r: o.compact ? 7 : 4.5 })); + return svg; + } + + // Overview block: on by default for the owner of a hive that uses + // Claude; hidden per browser from the Overview settings. + function renderSubscriptionGaugeOverview() { + const host = document.getElementById('overview-usage-gauge'); + if (!host) return; + const m = subscriptionGaugeCurrentModel(); + if (!subscriptionGaugeOwner() || !subscriptionGaugeOverviewEnabled() || !m || !m.usesClaude) { + host.hidden = true; + host.replaceChildren(); + return; + } + const el = (tag, cls, text) => { + const n = document.createElement(tag); + if (cls) n.className = cls; + if (text !== undefined) n.textContent = text; + return n; + }; + const head = el('div', 'overview-usage-gauge__head'); + head.appendChild(el('div', 'overview-usage-gauge__title', 'Claude usage left')); + const meta = []; + if (m.state === 'reading') { + if (m.capturedAt) meta.push('Measured ' + relativeTimeShort(m.capturedAt)); + if (m.planName) meta.push('Plan: ' + m.planName); + } + head.appendChild(el('div', 'overview-usage-gauge__meta', meta.join(' · '))); + const parts = [head]; + if (m.state !== 'reading') { + parts.push(el('div', 'overview-usage-gauge__note', 'No reading: ' + m.reason)); + } else { + if (m.outOfDate) parts.push(el('div', 'overview-usage-gauge__stale', 'This reading is out of date; treat it as a rough guide until hive measures again.')); + const grid = el('div', 'overview-usage-gauge__dials'); + m.dials.forEach((d) => { + const cell = el('div', 'overview-usage-gauge__dial'); + cell.appendChild(subscriptionGaugeDialSvg(d.pctLeft, { status: d.status, stale: m.outOfDate, label: d.label + ': ' + d.pctLeft + '% left' })); + const pct = el('div', 'overview-usage-gauge__pct', d.pctLeft + '% left'); + pct.setAttribute('data-status', m.outOfDate ? 'neutral' : d.status); + cell.appendChild(pct); + cell.appendChild(el('div', 'overview-usage-gauge__label', d.label)); + if (d.resetsAt) cell.appendChild(el('div', 'overview-usage-gauge__reset', 'Resets ' + formatDashboardDateTime(d.resetsAt))); + grid.appendChild(cell); + }); + parts.push(grid); + } + parts.push(el('div', 'overview-usage-gauge__note', (m.accountNote || 'Reflects the Claude account this hive is signed in with.') + ' This is the Claude plan, not the hive token budget.')); + host.replaceChildren.apply(host, parts); + host.hidden = false; + } + + // Opt-in header chip: a compact dial whose needle shows the limit closest + // to empty. It shares the Overview gauge's readings; no separate fetch. + function renderSubscriptionGaugeChip() { + const el = document.getElementById('oc-subscription-gauge'); + if (!el) return; + const dialHost = el.querySelector('.subscription-gauge-chip__dial'); + const labelEl = el.querySelector('.nav-chip__label'); + const m = subscriptionGaugeCurrentModel(); + if (!subscriptionGaugeOwner() || !subscriptionGaugeHeaderEnabled() || !m) { + el.hidden = true; + if (dialHost) dialHost.replaceChildren(); + return; + } + const lines = []; + if (m.state !== 'reading') { + if (dialHost) dialHost.replaceChildren(); + if (labelEl) labelEl.textContent = 'Claude: no reading'; + el.dataset.status = 'neutral'; + el.dataset.outOfDate = 'false'; + lines.push('No Claude usage reading: ' + m.reason); + } else { + const n = m.needle; + const label = 'Claude ' + n.pctLeft + '%' + (n.shortName ? ' · ' + n.shortName : ''); + if (dialHost) dialHost.replaceChildren(subscriptionGaugeDialSvg(n.pctLeft, { compact: true, status: n.status, stale: m.outOfDate, label: n.label + ': ' + n.pctLeft + '% left' })); + if (labelEl) labelEl.textContent = label; + el.dataset.status = m.outOfDate ? 'neutral' : m.worstStatus; + el.dataset.outOfDate = m.outOfDate ? 'true' : 'false'; + m.dials.forEach((d) => { + lines.push(d.label + ': ' + d.pctLeft + '% left' + (d.resetsAt ? ' · resets ' + formatDashboardDateTime(d.resetsAt) : '')); + }); + if (m.capturedAt) lines.push('Measured ' + relativeTimeShort(m.capturedAt) + (m.outOfDate ? ' (out of date)' : '')); + } + lines.push('Click to open the usage gauge on the Overview.'); + el.title = lines.join('\n'); + el.setAttribute('aria-label', (m.state === 'reading' ? 'Claude subscription: ' + m.needle.pctLeft + '% left in the ' + m.needle.label + ' limit' : 'Claude subscription: no reading') + '. Open the usage gauge on the Overview.'); + el.hidden = false; + } + function openSubscriptionGaugeOverview() { + if (typeof ocNavigate === 'function') ocNavigate('overview-section'); + const host = document.getElementById('overview-usage-gauge'); + if (host && !host.hidden) { + host.scrollIntoView({ block: 'start', behavior: 'smooth' }); + } else if (typeof showToast === 'function') { + showToast('The usage gauge is hidden on the Overview. Turn on "Show Claude usage gauge" in Overview settings to see every limit.', 'info'); + } + } diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index 310a469a9b..a9908261a7 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -130,6 +130,7 @@ js/contributors.js js/refresh.js js/scroll-stabilizer.js js/inception.js +js/subscription-gauge.js js/debug-logs.js js/tooltips-dialogs.js js/plan-agent-controls.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 1cccd9750b..15d51756ac 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -462,6 +462,11 @@ `[hidden] { display: none }`, so a hidden status chip otherwise paints as an empty outlined pill (#oc-providers when every provider is usable). */ .oc-providers-chip[hidden] { display: none; } + /* Opt-in Claude usage chip: a compact fuel dial plus its percentage. */ + .subscription-gauge-chip[hidden] { display: none; } + .subscription-gauge-chip__dial { display: inline-flex; align-items: center; } + .subscription-gauge-chip__dial:empty { display: none; } + .subscription-gauge-chip[data-out-of-date="true"] { border-style: dashed; } .oc-version-navbar-upgrade { --oc-version-bee-lane: 56px; position: relative; display: flex; align-items: center; min-width: 0; max-width: 100%; flex: 1 1 auto; overflow: visible; } .oc-version-navbar-upgrade .oc-version-hive { position: absolute; left: 0; top: 50%; transform: translateY(-50%); } .oc-version-navbar-upgrade .oc-version-honey-dot { width: 4px; height: 4px; } @@ -588,6 +593,7 @@ #acmm-badge { display: none; } .oc-health-badge .nav-chip__label, #oc-settings-btn .nav-chip__label, + #oc-subscription-gauge .nav-chip__label, #welcome-topbar-btn .nav-chip__label { display: none; } } @@ -3757,6 +3763,40 @@ .overview-pr-open { --slice-c: var(--band-open); } .overview-pr-draft { --slice-c: var(--band-draft); } + /* Claude subscription usage gauge: Overview block and fuel dials. */ + .overview-usage-gauge { display: flex; flex-direction: column; gap: var(--sp-3); margin: var(--sp-0) var(--sp-0) var(--sp-4); padding: var(--sp-4); border: 1px solid var(--border); border-radius: var(--r-lg); background: var(--surface-2); } + .overview-usage-gauge[hidden] { display: none; } + .overview-usage-gauge__head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: var(--sp-3); } + .overview-usage-gauge__title { font-size: var(--fs-md); font-weight: var(--fw-bold); } + .overview-usage-gauge__meta, .overview-usage-gauge__reset, .overview-usage-gauge__note { font-size: var(--fs-sm); color: var(--text-muted); } + .overview-usage-gauge__stale { font-size: var(--fs-sm); color: var(--status-warn); } + .overview-usage-gauge__dials { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: var(--sp-4); } + .overview-usage-gauge__dial { display: flex; flex-direction: column; align-items: center; gap: var(--sp-1); text-align: center; } + .overview-usage-gauge__label { font-weight: var(--fw-semibold); } + .overview-usage-gauge__pct { font-size: var(--fs-lg); font-weight: var(--fw-bold); color: var(--status-ok); } + .overview-usage-gauge__pct[data-status="warn"] { color: var(--status-warn); } + .overview-usage-gauge__pct[data-status="error"] { color: var(--status-error); } + .overview-usage-gauge__pct[data-status="neutral"] { color: var(--text-muted); } + .sg-dial { display: block; width: 150px; height: auto; overflow: visible; } + .sg-dial--compact { width: 34px; } + .sg-dial__track, .sg-dial__zone, .sg-dial__fill { fill: none; stroke-width: 7; } + .sg-dial--compact .sg-dial__track, .sg-dial--compact .sg-dial__zone, .sg-dial--compact .sg-dial__fill { stroke-width: 12; } + .sg-dial__track { stroke: var(--border); } + .sg-dial__zone--red { stroke: var(--status-error); stroke-opacity: 0.55; } + .sg-dial__zone--amber { stroke: var(--status-warn); stroke-opacity: 0.55; } + .sg-dial__fill { stroke: var(--status-ok); stroke-opacity: 0.35; } + .sg-dial__tick { fill: none; stroke: var(--text-muted); stroke-width: 1.6; } + .sg-dial__letter { fill: var(--text-muted); font-weight: var(--fw-bold); } + .sg-dial__letter--e { fill: var(--status-error); } + .sg-dial__needle { stroke: var(--status-ok); stroke-width: 3.5; stroke-linecap: round; } + .sg-dial--compact .sg-dial__needle { stroke-width: 7; } + .sg-dial__hub { fill: var(--status-ok); } + .sg-dial[data-status="warn"] .sg-dial__needle, .sg-dial[data-status="warn"] .sg-dial__fill { stroke: var(--status-warn); } + .sg-dial[data-status="warn"] .sg-dial__hub { fill: var(--status-warn); } + .sg-dial[data-status="error"] .sg-dial__needle, .sg-dial[data-status="error"] .sg-dial__fill { stroke: var(--status-error); } + .sg-dial[data-status="error"] .sg-dial__hub { fill: var(--status-error); } + .sg-dial[data-stale="true"] .sg-dial__needle, .sg-dial[data-stale="true"] .sg-dial__fill { stroke: var(--text-muted); } + .sg-dial[data-stale="true"] .sg-dial__hub { fill: var(--text-muted); } /* Sparklines */ .sparkline { display: inline-block; vertical-align: middle; margin-left: var(--sp-3); flex-shrink: 0; } .sparkline svg { display: block; overflow: visible; } @@ -8731,6 +8771,7 @@ <h2 class="config-title">🐝 Getting Started<span id="welcome-progress" class=" <!-- Agent Logs hidden (redundant with per-agent output) --> <button type="button" id="oc-navbar-upgrade" class="nav-chip nav-chip--status nav-chip--warn oc-navbar-upgrade" data-action="focusVersionBlock" title="Upgrade progress — jump to Version details" aria-label="Upgrade progress" hidden></button> <span class="nav-chip nav-chip--status oc-providers-chip" id="oc-providers" data-status="warn" role="status" title="AI providers: all usable" aria-label="AI providers: all usable" hidden></span> + <button type="button" class="nav-chip nav-chip--status subscription-gauge-chip" id="oc-subscription-gauge" data-action="openSubscriptionGaugeOverview" data-status="neutral" aria-label="Claude subscription usage left" hidden><span class="subscription-gauge-chip__dial" aria-hidden="true"></span><span class="nav-chip__label"></span></button> <button type="button" class="nav-chip nav-chip--status nav-chip--attention oc-waiting-on-you" id="oc-waiting-on-you" data-action="showWaitingOnYou" title="Nothing is waiting on you" aria-label="0 waiting on you — show in the activity monitor" hidden><span class="nav-chip__dot" aria-hidden="true">●</span> <span class="nav-chip__label">Waiting on you · 0</span></button> <span class="nav-chip nav-chip--status nav-chip--ok oc-health-badge" id="oc-health"><span class="nav-chip__dot" aria-hidden="true">●</span> <span class="nav-chip__label">Health OK</span></span> <button class="nav-chip nav-chip--primary" id="welcome-topbar-btn" style="display:none" data-action="openWelcomeDialog" title="Open the Getting Started guide" aria-label="Guide"><span class="nav-chip__icon" aria-hidden="true">🧭</span><span class="nav-chip__label">Guide</span></button> @@ -8827,6 +8868,7 @@ <h2 class="config-title" id="overview-chart-settings-title">Overview settings</h </div> </div> <div class="section-body"> + <div class="overview-usage-gauge" id="overview-usage-gauge" role="region" aria-label="Claude subscription usage" hidden></div> <div class="overview-card" id="overview-card"></div> </div> </div> @@ -34987,7 +35029,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> function overviewDefaultChartState() { const enabled = {}; OVERVIEW_CHART_TYPES.forEach(type => { enabled[type] = type === 'donut'; }); - return { enabled, carousel: false, intervalMs: OVERVIEW_CAROUSEL_DEFAULT_MS, transition: 'fade', duration: 'normal', active: { issue: 'donut', pr: 'donut' }, history: { issue: [], pr: [] }, labelMode: 'both', showKPIs: true, exportFormat: 'csv' }; + return { enabled, carousel: false, intervalMs: OVERVIEW_CAROUSEL_DEFAULT_MS, transition: 'fade', duration: 'normal', active: { issue: 'donut', pr: 'donut' }, history: { issue: [], pr: [] }, labelMode: 'both', showKPIs: true, showUsageGauge: true, exportFormat: 'csv' }; } function overviewChartsState() { @@ -35004,7 +35046,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const history = saved.history && typeof saved.history === 'object' ? saved.history : base.history; const labelMode = OVERVIEW_LABEL_MODES.includes(saved.labelMode) ? saved.labelMode : base.labelMode; const exportFormat = OVERVIEW_EXPORT_FORMATS.includes(saved.exportFormat) ? saved.exportFormat : base.exportFormat; - return { enabled, carousel: !!saved.carousel, intervalMs, transition, duration, active, history, labelMode, showKPIs: saved.showKPIs !== false, exportFormat }; + return { enabled, carousel: !!saved.carousel, intervalMs, transition, duration, active, history, labelMode, showKPIs: saved.showKPIs !== false, showUsageGauge: saved.showUsageGauge !== false, exportFormat }; } const OVERVIEW_KPI_WINDOWS = { '24h': 24 * 3600e3, '7d': 7 * 24 * 3600e3 }; @@ -35900,7 +35942,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const autoTitle = state.carousel ? 'Auto-play on — rotates every ' + Math.round(overviewCarouselDwellMs(state) / 1000) + 's through the ticked charts. Click to pause.' : 'Auto-play off — click to rotate through the ticked charts every ' + Math.round(overviewCarouselDwellMs(state) / 1000) + 's'; return `<div class="overview-settings-section" aria-label="${esc(label)} chart controls"><div class="overview-settings-card-title">${esc(label)}</div><div class="overview-chart-controls"><button type="button" class="overview-carousel-arrow" data-action="overviewCarouselPrev" data-arg0="${kind}" data-stop="1" title="Previous ${esc(label)} chart" aria-label="Previous ${esc(label)} chart">◀</button><label class="overview-settings-field">Chart type<select class="overview-chart-select" data-change-action="setOverviewChartType" data-arg0="${kind}" data-arg-types="s,v" title="${esc(label)} chart type">${overviewChartTypeOptions(active)}</select></label><button type="button" class="overview-carousel-arrow" data-action="overviewCarouselNext" data-arg0="${kind}" data-stop="1" title="Next ${esc(label)} chart" aria-label="Next ${esc(label)} chart">▶</button>${renderOverviewDots(enabled, active)}<span class="overview-chart-controls-sep"></span><button type="button" class="overview-carousel-arrow" data-action="toggleOverviewCarousel" data-stop="1" aria-pressed="${state.carousel ? 'true' : 'false'}" title="${esc(autoTitle)}" aria-label="${esc(autoTitle)}">${state.carousel ? '⏸ auto' : '⏵ auto'}</button><label class="overview-settings-field">Transition<select class="overview-chart-select" data-change-action="setOverviewTransition" data-arg-types="v" title="Transition between charts">${overviewTransitionOptions(state.transition)}</select></label></div></div>`; }; - setIfChanged(el, `<div class="overview-settings-section"><div class="overview-settings-card-title">Overview charts</div><div class="overview-settings-grid">${typeRows}</div><div class="overview-settings-grid"><label class="overview-settings-field">Interval (seconds)<input type="number" min="5" max="300" step="5" value="${seconds}" data-change-action="setOverviewCarouselInterval" data-arg-types="N"></label><label class="overview-settings-field">Duration<select data-change-action="setOverviewDuration" data-arg-types="v">${durationOptions}</select></label><label class="overview-settings-field">Export format<select data-change-action="setOverviewExportFormat" data-arg-types="v">${exportOptions}</select></label><label class="overview-settings-field">Donut labels<select data-change-action="setOverviewLabelMode" data-arg-types="v">${labelOptions}</select></label><label class="overview-settings-row"><input type="checkbox" data-change-action="setOverviewKPIs" data-arg-types="c"${state.showKPIs ? ' checked' : ''}>Show KPI strip</label></div><div class="overview-settings-links"><button type="button" class="hv-btn btn-sm" data-action="resetOverviewSettings" data-stop="1">Reset to defaults</button></div></div>${overviewRepoFilterHTML(_overviewLastRepos || [])}${cardControls('issue', 'Issues')}${cardControls('pr', 'PRs')}`); + setIfChanged(el, `<div class="overview-settings-section"><div class="overview-settings-card-title">Overview charts</div><div class="overview-settings-grid">${typeRows}</div><div class="overview-settings-grid"><label class="overview-settings-field">Interval (seconds)<input type="number" min="5" max="300" step="5" value="${seconds}" data-change-action="setOverviewCarouselInterval" data-arg-types="N"></label><label class="overview-settings-field">Duration<select data-change-action="setOverviewDuration" data-arg-types="v">${durationOptions}</select></label><label class="overview-settings-field">Export format<select data-change-action="setOverviewExportFormat" data-arg-types="v">${exportOptions}</select></label><label class="overview-settings-field">Donut labels<select data-change-action="setOverviewLabelMode" data-arg-types="v">${labelOptions}</select></label><label class="overview-settings-row"><input type="checkbox" data-change-action="setOverviewKPIs" data-arg-types="c"${state.showKPIs ? ' checked' : ''}>Show KPI strip</label>${typeof subscriptionGaugeOwner === 'function' && subscriptionGaugeOwner() ? `<label class="overview-settings-row"><input type="checkbox" data-change-action="setOverviewUsageGauge" data-arg-types="c"${state.showUsageGauge ? ' checked' : ''}>Show Claude usage gauge</label>` : ''}</div><div class="overview-settings-links"><button type="button" class="hv-btn btn-sm" data-action="resetOverviewSettings" data-stop="1">Reset to defaults</button></div></div>${overviewRepoFilterHTML(_overviewLastRepos || [])}${cardControls('issue', 'Issues')}${cardControls('pr', 'PRs')}`); } function rerenderOverviewChartsFromState(state) { @@ -35972,6 +36014,13 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> rerenderOverviewChartsFromState(state); } + function setOverviewUsageGauge(checked) { + const state = overviewChartsState(); + state.showUsageGauge = !!checked; + rerenderOverviewChartsFromState(state); + if (typeof subscriptionGaugeRefresh === 'function') subscriptionGaugeRefresh(); + } + function setOverviewRepoFilter(repo, checked) { const all = overviewAllRepoNames(_overviewLastRepos || []); const selected = new Set(overviewSelectedRepoNames(_overviewLastRepos || [])); @@ -48978,6 +49027,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> // Update Light health badge with hover detail renderNavbarHealthBadge(document.getElementById('oc-health'), data); renderProviderStrip(document.getElementById('oc-providers'), data.providers); + if (typeof subscriptionGaugeRefresh === 'function') subscriptionGaugeRefresh(); renderGhRateLimits(data.ghRateLimits || {}); renderGitHubAppBanner(data); renderRepoTargetBanner(data); @@ -50407,6 +50457,306 @@ <h4>GitHub Projects sync</h4> return `<div class="quality-stat-card"><div class="quality-stat-title">Quality stats</div>${rows}</div>`; } + // Claude subscription usage gauge. subscriptionGaugeModel is the single + // pure decision point: it turns a /api/providers/headroom response into + // the dials, the needle and the no-reading reason that the Overview gauge + // and the header chip render. No DOM, no fetch, so node tests can run it. + function subscriptionGaugeModel(resp, nowMs) { + const OUT_OF_DATE_MS = 15 * 60 * 1000; + const REASONS = { + not_installed: 'Claude Code is not installed on this hive.', + no_credentials: 'This hive is not signed in to Claude.', + rate_limited: 'Claude rate-limited the usage check; hive will try again.', + timeout: 'The usage check timed out; hive will try again.', + unrecognized_schema: 'Claude changed its usage format, so hive could not read it.', + probe_failed: 'The usage check failed; hive will try again.', + }; + const NOT_USED = ['not_installed', 'no_credentials']; + const validTime = (v) => { + const ms = Date.parse(v || ''); + // Go's zero time.Time serializes as 0001-01-01: treat it as unknown. + return Number.isFinite(ms) && ms > 0 ? ms : null; + }; + const statusFor = (pct) => (pct <= 10 ? 'error' : pct <= 25 ? 'warn' : 'ok'); + const describe = (kind, id, mins, scope) => { + const k = String(kind || id || '').toLowerCase(); + const i = String(id || '').toLowerCase(); + const sc = scope || {}; + // A weekly limit that counts only some models: the builtin source + // sends kind weekly_scoped; ccleft sends kind weekly with id + // seven_day_<model> and scope.scope set (binding "false"). + const scoped = k === 'weekly_scoped' || ((k === 'weekly' || k === 'seven_day') && + (!!sc.scope || /^(seven_day|weekly)_(?!all$)./.test(i))); + if (scoped) { + let model = sc.model || sc.model_family || sc.scope || sc.name || ''; + if (/^scoped$/i.test(String(model))) model = ''; + return { label: model ? 'Weekly (' + model + ')' : 'Weekly (model-specific)', shortName: 'model' }; + } + if (k === 'weekly' || k === 'weekly_all' || k === 'seven_day') return { label: 'Weekly', shortName: '' }; + if (k === 'session' || k === 'five_hour') return { label: '5-hour', shortName: '5-hour' }; + if (k === 'daily') return { label: 'Daily', shortName: 'daily' }; + if (k === 'monthly') return { label: 'Monthly', shortName: 'monthly' }; + if (mins > 0 && mins % 1440 === 0) return { label: (mins / 1440) + '-day', shortName: (mins / 1440) + '-day' }; + if (mins > 0) return { label: Math.round(mins / 60) + '-hour', shortName: Math.round(mins / 60) + '-hour' }; + return { label: 'Limit', shortName: 'limit' }; + }; + const providers = resp && Array.isArray(resp.providers) ? resp.providers : []; + const reading = providers.find((p) => p && p.provider === 'anthropic'); + if (!reading) { + return { state: 'no-reading', usesClaude: false, cause: 'absent', reason: 'Usage measurement is not running on this hive.' }; + } + const cause = String(reading.probe_error_cause || ''); + const limits = Array.isArray(reading.limits) ? reading.limits : []; + const dials = []; + // Paid credit pools and prepaid balances are not plan limits. + const NOT_LIMITS = ['credits', 'balance']; + limits.forEach((l, i) => { + if (!l || typeof l.pct_remaining !== 'number' || !Number.isFinite(l.pct_remaining)) return; + if (NOT_LIMITS.includes(String(l.kind || '').toLowerCase())) return; + const mins = Number(l.duration_mins) || 0; + const d = describe(l.kind, l.id, mins, l.scope); + const pctLeft = Math.max(0, Math.min(100, Math.round(l.pct_remaining))); + const reset = validTime(l.resets_at); + dials.push({ + id: String(l.id || l.kind || ''), + kind: String(l.kind || ''), + label: d.label, + shortName: d.shortName, + pctLeft: pctLeft, + resetsAt: reset === null ? '' : new Date(reset).toISOString(), + durationMins: mins, + status: statusFor(pctLeft), + order: i, + }); + }); + if (!dials.length) { + return { + state: 'no-reading', + usesClaude: !NOT_USED.includes(cause), + cause: cause || 'empty', + reason: REASONS[cause] || 'No usage reading yet.', + }; + } + // Longest window first; among equal lengths the all-models weekly + // limit leads, then the provider's own order. + const rank = (dial) => (dial.shortName === '' ? 0 : 1); + dials.sort((a, b) => { + const am = a.durationMins > 0 ? a.durationMins : -1; + const bm = b.durationMins > 0 ? b.durationMins : -1; + if (am !== bm) return bm - am; + if (rank(a) !== rank(b)) return rank(a) - rank(b); + return a.order - b.order; + }); + // The needle follows the window with the least left; ties keep the + // earlier (longer) window. + let needle = dials[0]; + dials.forEach((dial) => { if (dial.pctLeft < needle.pctLeft) needle = dial; }); + const captured = validTime(reading.captured_at); + const ageMs = captured === null || !Number.isFinite(nowMs) ? null : Math.max(0, nowMs - captured); + return { + state: 'reading', + usesClaude: true, + cause: cause, + dials: dials.map((dial) => ({ + id: dial.id, kind: dial.kind, label: dial.label, shortName: dial.shortName, + pctLeft: dial.pctLeft, resetsAt: dial.resetsAt, durationMins: dial.durationMins, status: dial.status, + })), + needle: { id: needle.id, kind: needle.kind, label: needle.label, shortName: needle.shortName, pctLeft: needle.pctLeft, status: needle.status }, + worstStatus: needle.status, + capturedAt: captured === null ? '' : new Date(captured).toISOString(), + ageMs: ageMs, + outOfDate: !!reading.stale || (ageMs !== null && ageMs > OUT_OF_DATE_MS), + planName: String(reading.plan_type || ''), + accountNote: 'Reflects the Claude account this hive is signed in with.', + }; + } + + // Shared loader. The headroom endpoint is owner-only and the fetch + // wrapper drops a stored token on any 401/403 from /api/, so nothing is + // fetched until the viewer is known to be the owner. The endpoint serves + // stored readings: polling it never triggers a measurement. + let _subscriptionGaugeData = null; + const SUBSCRIPTION_GAUGE_POLL_MS = 60000; + function subscriptionGaugeOwner() { + return typeof dashboardRoleAtLeast === 'function' && dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + } + function subscriptionGaugeOverviewEnabled() { + try { return typeof overviewChartsState !== 'function' || overviewChartsState().showUsageGauge !== false; } catch (e) { return true; } + } + function subscriptionGaugeHeaderEnabled() { + return typeof dashboardFeatureEnabled === 'function' && dashboardFeatureEnabled('subscription_gauge_header'); + } + function subscriptionGaugeWanted() { + return subscriptionGaugeOwner() && (subscriptionGaugeOverviewEnabled() || subscriptionGaugeHeaderEnabled()); + } + function subscriptionGaugeCurrentModel() { + return _subscriptionGaugeData ? subscriptionGaugeModel(_subscriptionGaugeData, Date.now()) : null; + } + function subscriptionGaugeRefresh() { + if (!subscriptionGaugeWanted()) { + dashboardClearInterval('subscription-gauge'); + } else if (!DASHBOARD_INTERVALS['subscription-gauge']) { + dashboardSetInterval('subscription-gauge', subscriptionGaugeFetch, SUBSCRIPTION_GAUGE_POLL_MS); + subscriptionGaugeFetch(); + } + renderSubscriptionGaugeSurfaces(); + } + async function subscriptionGaugeFetch() { + if (!subscriptionGaugeWanted()) return; + try { + const res = await fetch('/api/providers/headroom'); + if (!res.ok) return; + _subscriptionGaugeData = await res.json(); + } catch (e) { + return; + } + renderSubscriptionGaugeSurfaces(); + } + function renderSubscriptionGaugeSurfaces() { + renderSubscriptionGaugeOverview(); + if (typeof renderSubscriptionGaugeChip === 'function') renderSubscriptionGaugeChip(); + } + + // An E-to-F fuel dial drawn with DOM calls: numbers only, colours from + // CSS classes keyed on data-status, no inline styles. + function subscriptionGaugeDialSvg(pctLeft, opts) { + const o = opts || {}; + const NS = 'http://www.w3.org/2000/svg'; + const cx = 50, cy = 54, r = 42; + const v = Math.max(0, Math.min(100, Number(pctLeft) || 0)); + const pt = (f, rr) => { + const a = Math.PI * (1 - f); + return (cx + rr * Math.cos(a)).toFixed(2) + ' ' + (cy - rr * Math.sin(a)).toFixed(2); + }; + const arc = (a, b) => 'M ' + pt(a, r) + ' A ' + r + ' ' + r + ' 0 0 1 ' + pt(b, r); + const node = (tag, attrs) => { + const el = document.createElementNS(NS, tag); + Object.keys(attrs).forEach((k) => el.setAttribute(k, String(attrs[k]))); + return el; + }; + const svg = node('svg', { + viewBox: '0 0 100 62', + class: 'sg-dial' + (o.compact ? ' sg-dial--compact' : ''), + 'data-status': o.status || (v <= 10 ? 'error' : v <= 25 ? 'warn' : 'ok'), + 'data-stale': o.stale ? 'true' : 'false', + role: 'img', + 'aria-label': o.label || (v + '% left'), + }); + svg.appendChild(node('path', { class: 'sg-dial__track', d: arc(0, 1) })); + svg.appendChild(node('path', { class: 'sg-dial__zone sg-dial__zone--red', d: arc(0, 0.10) })); + svg.appendChild(node('path', { class: 'sg-dial__zone sg-dial__zone--amber', d: arc(0.10, 0.25) })); + if (v > 0.5) svg.appendChild(node('path', { class: 'sg-dial__fill', d: arc(0, v / 100) })); + if (!o.compact) { + let ticks = ''; + [0, 0.25, 0.5, 0.75, 1].forEach((f) => { ticks += 'M ' + pt(f, r - 9) + ' L ' + pt(f, r - 14) + ' '; }); + svg.appendChild(node('path', { class: 'sg-dial__tick', d: ticks.trim() })); + const e = node('text', { class: 'sg-dial__letter sg-dial__letter--e', x: 6, y: 60, 'font-size': 11 }); + e.textContent = 'E'; + const f = node('text', { class: 'sg-dial__letter', x: 87, y: 60, 'font-size': 11 }); + f.textContent = 'F'; + svg.appendChild(e); + svg.appendChild(f); + } + const tip = pt(v / 100, r - 5).split(' '); + svg.appendChild(node('line', { class: 'sg-dial__needle', x1: cx, y1: cy, x2: tip[0], y2: tip[1] })); + svg.appendChild(node('circle', { class: 'sg-dial__hub', cx: cx, cy: cy, r: o.compact ? 7 : 4.5 })); + return svg; + } + + // Overview block: on by default for the owner of a hive that uses + // Claude; hidden per browser from the Overview settings. + function renderSubscriptionGaugeOverview() { + const host = document.getElementById('overview-usage-gauge'); + if (!host) return; + const m = subscriptionGaugeCurrentModel(); + if (!subscriptionGaugeOwner() || !subscriptionGaugeOverviewEnabled() || !m || !m.usesClaude) { + host.hidden = true; + host.replaceChildren(); + return; + } + const el = (tag, cls, text) => { + const n = document.createElement(tag); + if (cls) n.className = cls; + if (text !== undefined) n.textContent = text; + return n; + }; + const head = el('div', 'overview-usage-gauge__head'); + head.appendChild(el('div', 'overview-usage-gauge__title', 'Claude usage left')); + const meta = []; + if (m.state === 'reading') { + if (m.capturedAt) meta.push('Measured ' + relativeTimeShort(m.capturedAt)); + if (m.planName) meta.push('Plan: ' + m.planName); + } + head.appendChild(el('div', 'overview-usage-gauge__meta', meta.join(' · '))); + const parts = [head]; + if (m.state !== 'reading') { + parts.push(el('div', 'overview-usage-gauge__note', 'No reading: ' + m.reason)); + } else { + if (m.outOfDate) parts.push(el('div', 'overview-usage-gauge__stale', 'This reading is out of date; treat it as a rough guide until hive measures again.')); + const grid = el('div', 'overview-usage-gauge__dials'); + m.dials.forEach((d) => { + const cell = el('div', 'overview-usage-gauge__dial'); + cell.appendChild(subscriptionGaugeDialSvg(d.pctLeft, { status: d.status, stale: m.outOfDate, label: d.label + ': ' + d.pctLeft + '% left' })); + const pct = el('div', 'overview-usage-gauge__pct', d.pctLeft + '% left'); + pct.setAttribute('data-status', m.outOfDate ? 'neutral' : d.status); + cell.appendChild(pct); + cell.appendChild(el('div', 'overview-usage-gauge__label', d.label)); + if (d.resetsAt) cell.appendChild(el('div', 'overview-usage-gauge__reset', 'Resets ' + formatDashboardDateTime(d.resetsAt))); + grid.appendChild(cell); + }); + parts.push(grid); + } + parts.push(el('div', 'overview-usage-gauge__note', (m.accountNote || 'Reflects the Claude account this hive is signed in with.') + ' This is the Claude plan, not the hive token budget.')); + host.replaceChildren.apply(host, parts); + host.hidden = false; + } + + // Opt-in header chip: a compact dial whose needle shows the limit closest + // to empty. It shares the Overview gauge's readings; no separate fetch. + function renderSubscriptionGaugeChip() { + const el = document.getElementById('oc-subscription-gauge'); + if (!el) return; + const dialHost = el.querySelector('.subscription-gauge-chip__dial'); + const labelEl = el.querySelector('.nav-chip__label'); + const m = subscriptionGaugeCurrentModel(); + if (!subscriptionGaugeOwner() || !subscriptionGaugeHeaderEnabled() || !m) { + el.hidden = true; + if (dialHost) dialHost.replaceChildren(); + return; + } + const lines = []; + if (m.state !== 'reading') { + if (dialHost) dialHost.replaceChildren(); + if (labelEl) labelEl.textContent = 'Claude: no reading'; + el.dataset.status = 'neutral'; + el.dataset.outOfDate = 'false'; + lines.push('No Claude usage reading: ' + m.reason); + } else { + const n = m.needle; + const label = 'Claude ' + n.pctLeft + '%' + (n.shortName ? ' · ' + n.shortName : ''); + if (dialHost) dialHost.replaceChildren(subscriptionGaugeDialSvg(n.pctLeft, { compact: true, status: n.status, stale: m.outOfDate, label: n.label + ': ' + n.pctLeft + '% left' })); + if (labelEl) labelEl.textContent = label; + el.dataset.status = m.outOfDate ? 'neutral' : m.worstStatus; + el.dataset.outOfDate = m.outOfDate ? 'true' : 'false'; + m.dials.forEach((d) => { + lines.push(d.label + ': ' + d.pctLeft + '% left' + (d.resetsAt ? ' · resets ' + formatDashboardDateTime(d.resetsAt) : '')); + }); + if (m.capturedAt) lines.push('Measured ' + relativeTimeShort(m.capturedAt) + (m.outOfDate ? ' (out of date)' : '')); + } + lines.push('Click to open the usage gauge on the Overview.'); + el.title = lines.join('\n'); + el.setAttribute('aria-label', (m.state === 'reading' ? 'Claude subscription: ' + m.needle.pctLeft + '% left in the ' + m.needle.label + ' limit' : 'Claude subscription: no reading') + '. Open the usage gauge on the Overview.'); + el.hidden = false; + } + function openSubscriptionGaugeOverview() { + if (typeof ocNavigate === 'function') ocNavigate('overview-section'); + const host = document.getElementById('overview-usage-gauge'); + if (host && !host.hidden) { + host.scrollIntoView({ block: 'start', behavior: 'smooth' }); + } else if (typeof showToast === 'function') { + showToast('The usage gauge is hidden on the Overview. Turn on "Show Claude usage gauge" in Overview settings to see every limit.', 'info'); + } + } // ── Debug section ── const DEBUG_REFRESH_MS = 10000; const GH_RATE_CRITICAL_REMAINING_PCT = 10; @@ -55810,6 +56160,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> { tab: 'Features', section: 'Work intake & issue lifecycle', label: 'Stall replan', key: 'governor.replan.enabled', text: 'stall replan approved plans architect interval threshold max replans' }, { tab: 'Advisory', section: 'Queue health', label: 'Queue health thresholds', key: 'governor.advisory.queue_health.blocked_pr_pct', text: 'advisory queue health blocked needs human stale lane share check share thresholds' }, { tab: 'Dashboard', section: 'Dashboard runtime', label: 'Experiment Lab', key: 'dashboard.experiment_lab', text: 'experiment_lab Experiment Lab dashboard feature experimental planner' }, + { tab: 'Dashboard', section: 'Dashboard runtime', label: 'Subscription usage gauge in header', key: 'dashboard.subscription_gauge_header', text: 'subscription_gauge_header subscription usage gauge fuel header chip claude usage claude plan weekly 5-hour limit left remaining quota headroom' }, { tab: 'Project Observability', section: 'Target platforms', label: 'Project observability targets', key: 'governor.project_observability.open_source[]', text: 'project observability telemetry operations open source kube native commercial references endpoint env credential secret' }, { tab: 'Features', section: 'Work intake & issue lifecycle', label: 'Question auto-close', key: 'governor.question_autoclose.enabled', text: 'question auto close answered questions labels human objection window' }, { tab: 'Logging', section: 'Logging', label: 'Kick log source', key: 'logging.kick_log_source', text: 'kick log source live file audit' }, @@ -57728,6 +58079,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> '<div class="config-field"><label>Agent poll interval (seconds) ' + yamlHint('dashboard.agent_poll_interval_s') + '</label><input type="number" min="1" value="' + esc(String(dash.agent_poll_interval_s || 0)) + '" data-change-action="markDirty" data-arg0="dashboard" data-arg1="agent_poll_interval_s" data-arg-types="s,s,N"></div>' + '</div>' + '<div class="config-toggle"><div class="config-toggle-switch ' + (dash.experiment_lab ? 'on' : '') + '" data-action="toggleConfigSwitch" data-section="dashboard" data-key="experiment_lab"></div><span class="config-toggle-label">Experiment Lab ' + yamlHint('dashboard.experiment_lab') + '</span></div>' + + '<div class="config-toggle"><div class="config-toggle-switch ' + (dash.subscription_gauge_header ? 'on' : '') + '" data-action="toggleConfigSwitch" data-section="dashboard" data-key="subscription_gauge_header"></div><span class="config-toggle-label">Also show the usage gauge in the header ' + yamlHint('dashboard.subscription_gauge_header') + ' ' + settingHelpMark('settings', 'dashboard', 'Usage gauge in the header', 'The Claude usage fuel gauge is on the Overview by default for the owner. This also shows it as a small chip in the header on every page. It reads the usage measurements hive already makes, so it adds no calls to Claude. Only the hive owner sees it. This is the Claude plan this hive is signed in with, not the hive token budget. Off by default.') + '</span></div>' + '<div class="config-toggle"><div class="config-toggle-switch ' + (dash.hub_proxied ? 'on' : '') + '" data-action="toggleConfigSwitch" data-section="dashboard" data-key="hub_proxied"></div><span class="config-toggle-label">Hub-proxied auth ' + yamlHint('dashboard.hub_proxied') + ' <span class="text-muted">Managed by hub on hosted hives.</span></span></div>' + '<div class="config-field"><label>Dashboard auth token ' + yamlHint('dashboard.auth_token') + '</label><input type="password" autocomplete="new-password" placeholder="' + esc(authTokenText) + '" data-change-action="markDirty" data-arg0="dashboard" data-arg1="auth_token" data-arg-types="s,s,v"><div class="config-help">Write-only; the current token is never echoed.</div></div>' + '<h3>Access allowlist</h3>' + @@ -74034,6 +74386,7 @@ <h4>${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}</h4 const repaintTracker = hiveAuthResolved(); if (username) window._hiveUser = username; if (role) window._hiveRole = role; + if (typeof subscriptionGaugeRefresh === 'function') subscriptionGaugeRefresh(); const banner = document.getElementById('gh-auth-banner'); const topBtn = document.getElementById('oc-gh-login-btn'); const avatarWrap = document.getElementById('oc-gh-avatar-wrap'); diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 18be7c3f88..6433f6cba7 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -576,8 +576,9 @@ func buildFeatures(cfg *config.Config) FrontendFeatures { // without going through the loader. enabled := cfg.Dashboard.ExperimentLab || cfg.Dashboard.StrategyLab return FrontendFeatures{ - ExperimentLab: enabled, - StrategyLab: enabled, + ExperimentLab: enabled, + StrategyLab: enabled, + SubscriptionGaugeHeader: cfg.Dashboard.SubscriptionGaugeHeader, } } diff --git a/src/pkg/dashboard/status_builder_test.go b/src/pkg/dashboard/status_builder_test.go index fdc9cde4ce..af261d4dbf 100644 --- a/src/pkg/dashboard/status_builder_test.go +++ b/src/pkg/dashboard/status_builder_test.go @@ -1646,3 +1646,34 @@ func TestMarkStatusStaleRepublishesLastGoodWithMarker(t *testing.T) { t.Fatal("same marker must not republish") } } + +// TestBuildFeaturesSubscriptionGaugeHeader checks that the status frame's feature +// flags carry dashboard.subscription_gauge_header and nothing derived from usage. +func TestBuildFeaturesSubscriptionGaugeHeader(t *testing.T) { + if buildFeatures(nil).SubscriptionGaugeHeader { + t.Fatal("nil config reported subscription_gauge_header = true") + } + cfg := &config.Config{} + if buildFeatures(cfg).SubscriptionGaugeHeader { + t.Fatal("default config reported subscription_gauge_header = true") + } + cfg.Dashboard.SubscriptionGaugeHeader = true + f := buildFeatures(cfg) + if !f.SubscriptionGaugeHeader { + t.Fatal("subscription_gauge_header = false with dashboard.subscription_gauge_header enabled") + } + if f.ExperimentLab || f.StrategyLab { + t.Fatal("subscription_gauge_header leaked into experiment_lab/strategy_lab") + } + raw, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m["subscription_gauge_header"] != true { + t.Fatalf("features JSON = %s, want subscription_gauge_header:true", raw) + } +} diff --git a/src/pkg/dashboard/subscription_gauge.test.js b/src/pkg/dashboard/subscription_gauge.test.js new file mode 100644 index 0000000000..d6ae7ab981 --- /dev/null +++ b/src/pkg/dashboard/subscription_gauge.test.js @@ -0,0 +1,171 @@ +// Run the shipped subscription-gauge view-model against recorded +// /api/providers/headroom shapes. Pure function: no DOM, timers or network. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +// Values built inside the vm realm carry its prototypes; compare plain copies. +const plain = value => JSON.parse(JSON.stringify(value)); + +function loadModel() { + const src = fs.readFileSync(path.join(__dirname, 'static-src/js/subscription-gauge.js'), 'utf8'); + const found = src.match(/function subscriptionGaugeModel\([^\n]*\) \{[\s\S]*?\n \}/); + assert.ok(found, 'subscriptionGaugeModel'); + const ctx = vm.createContext({}); + vm.runInContext(found[0] + '\nthis.subscriptionGaugeModel = subscriptionGaugeModel;', ctx); + return ctx.subscriptionGaugeModel; +} +const model = loadModel(); + +const NOW = Date.parse('2026-11-13T18:12:00Z'); +const CAPTURED = '2026-11-13T18:09:00Z'; +const ZERO = '0001-01-01T00:00:00Z'; +function limit(id, kind, left, mins, extra) { + return Object.assign({ id, kind, percent_used: 100 - left, pct_remaining: left, resets_at: '2026-11-16T07:00:00Z', duration_mins: mins }, extra || {}); +} +function resp(anthropic, others) { + return { providers: (others || []).concat(anthropic ? [Object.assign({ provider: 'anthropic', available: true, captured_at: CAPTURED }, anthropic)] : []), enabled: false, readings: true }; +} + +test('weekly + 5-hour gives two dials, weekly first, needle on the lower', () => { + const m = model(resp({ limits: [limit('session', 'session', 30, 300), limit('weekly_all', 'weekly', 62, 10080)] }), NOW); + assert.equal(m.state, 'reading'); + assert.deepEqual(plain(m.dials.map(d => d.label)), ['Weekly', '5-hour']); + assert.equal(m.needle.label, '5-hour'); + assert.equal(m.needle.shortName, '5-hour'); + assert.equal(m.needle.pctLeft, 30); +}); + +test('weekly + 5-hour with weekly lower puts the needle on weekly with no suffix', () => { + const m = model(resp({ limits: [limit('weekly_all', 'weekly', 40, 10080), limit('session', 'session', 90, 300)] }), NOW); + assert.equal(m.needle.label, 'Weekly'); + assert.equal(m.needle.shortName, ''); +}); + +test('weekly-only reading gives exactly one dial and no placeholder', () => { + const m = model(resp({ limits: [limit('weekly_all', 'weekly', 47, 10080)] }), NOW); + assert.equal(m.dials.length, 1); + assert.equal(m.dials[0].label, 'Weekly'); +}); + +test('model-specific weekly is its own labelled dial and drives the needle when lowest', () => { + const m = model(resp({ limits: [limit('weekly_scoped', 'weekly_scoped', 22, 10080), limit('weekly_all', 'weekly', 70, 10080), limit('session', 'session', 88, 300)] }), NOW); + assert.deepEqual(plain(m.dials.map(d => d.label)), ['Weekly', 'Weekly (model-specific)', '5-hour']); + assert.equal(m.needle.shortName, 'model'); + assert.equal(m.needle.pctLeft, 22); + const named = model(resp({ limits: [limit('weekly_scoped', 'weekly_scoped', 50, 10080, { scope: { model: 'Opus' } })] }), NOW); + assert.equal(named.dials[0].label, 'Weekly (Opus)'); +}); + +test('ties keep the needle on the longer window', () => { + const m = model(resp({ limits: [limit('session', 'session', 50, 300), limit('weekly_all', 'weekly', 50, 10080)] }), NOW); + assert.equal(m.needle.label, 'Weekly'); +}); + +test('status thresholds: normal above 25, warn at 25 or less, error at 10 or less', () => { + const st = left => model(resp({ limits: [limit('weekly_all', 'weekly', left, 10080)] }), NOW).dials[0].status; + assert.equal(st(26), 'ok'); + assert.equal(st(25), 'warn'); + assert.equal(st(11), 'warn'); + assert.equal(st(10), 'error'); + assert.equal(st(0), 'error'); +}); + +test('percent left matches the reading to the whole percent and is clamped', () => { + const m = model(resp({ limits: [limit('weekly_all', 'weekly', 61.6, 10080), limit('session', 'session', 140, 300)] }), NOW); + assert.equal(m.dials[0].pctLeft, 62); + assert.equal(m.dials[1].pctLeft, 100); +}); + +test('out of date when stale, or captured more than 15 minutes ago; age from captured_at', () => { + const fresh = model(resp({ limits: [limit('weekly_all', 'weekly', 60, 10080)] }), NOW); + assert.equal(fresh.outOfDate, false); + assert.equal(fresh.ageMs, 3 * 60 * 1000); + const stale = model(resp({ stale: true, limits: [limit('weekly_all', 'weekly', 60, 10080)] }), NOW); + assert.equal(stale.outOfDate, true); + const old = model(resp({ captured_at: '2026-11-13T17:56:00Z', limits: [limit('weekly_all', 'weekly', 60, 10080)] }), NOW); + assert.equal(old.outOfDate, true); + // The response's own updated_at never decides age. + const r = resp({ limits: [limit('weekly_all', 'weekly', 60, 10080)] }); + r.updated_at = new Date(NOW).toISOString(); + assert.equal(model(r, NOW + 20 * 60 * 1000).outOfDate, true); +}); + +test('zero reset and capture times are treated as unknown', () => { + const m = model(resp({ captured_at: ZERO, limits: [limit('weekly_all', 'weekly', 60, 10080, { resets_at: ZERO })] }), NOW); + assert.equal(m.dials[0].resetsAt, ''); + assert.equal(m.capturedAt, ''); + assert.equal(m.ageMs, null); + assert.equal(m.outOfDate, false); +}); + +test('every probe cause gives a no-reading state with a reason and no percentage', () => { + for (const cause of ['not_installed', 'no_credentials', 'rate_limited', 'timeout', 'unrecognized_schema', 'probe_failed']) { + const m = model(resp({ available: true, pct_remaining: 0, probe_error_cause: cause, probe_error: 'x' }), NOW); + assert.equal(m.state, 'no-reading', cause); + assert.equal(m.cause, cause); + assert.ok(m.reason.length > 0, cause); + assert.equal(m.dials, undefined, cause); + assert.equal(m.needle, undefined, cause); + } + assert.equal(model(resp({ probe_error_cause: 'no_credentials' }), NOW).usesClaude, false); + assert.equal(model(resp({ probe_error_cause: 'not_installed' }), NOW).usesClaude, false); + assert.equal(model(resp({ probe_error_cause: 'rate_limited' }), NOW).usesClaude, true); +}); + +test('no anthropic reading means measurement is not running and Claude is not used', () => { + const m = model(resp(null, [{ provider: 'google', limits: [limit('gemini-weekly', 'weekly', 54, 10080)] }]), NOW); + assert.equal(m.state, 'no-reading'); + assert.equal(m.usesClaude, false); + assert.match(m.reason, /not running/); + assert.equal(model(null, NOW).state, 'no-reading'); + assert.equal(model({}, NOW).state, 'no-reading'); +}); + +test('other providers never leak into the Claude dials', () => { + const m = model(resp({ limits: [limit('weekly_all', 'weekly', 76, 10080)] }, [ + { provider: 'google', limits: [limit('gemini-argon-wtu-weekly', 'weekly', 1, 10080)] }, + { provider: 'openai', probe_error_cause: 'no_credentials' }, + ]), NOW); + assert.equal(m.dials.length, 1); + assert.equal(m.needle.pctLeft, 76); +}); + +test('plan name only when the reading carries one', () => { + assert.equal(model(resp({ limits: [limit('weekly_all', 'weekly', 60, 10080)] }), NOW).planName, ''); + assert.equal(model(resp({ plan_type: 'max', limits: [limit('weekly_all', 'weekly', 60, 10080)] }), NOW).planName, 'max'); +}); + +test('live shape from a v6 hive with rotation off (weekly + model-scoped, no 5-hour)', () => { + const m = model(resp({ pct_remaining: 70, limits: [ + { id: 'weekly_all', kind: 'weekly', percent_used: 24, pct_remaining: 76, resets_at: '2026-11-16T07:00:00.183473Z', duration_mins: 10080 }, + { id: 'weekly_scoped', kind: 'weekly_scoped', percent_used: 30, pct_remaining: 70, resets_at: '2026-11-16T07:00:00.18362Z', duration_mins: 10080 }, + ] }), NOW); + assert.deepEqual(plain(m.dials.map(d => [d.label, d.pctLeft])), [['Weekly', 76], ['Weekly (model-specific)', 70]]); + assert.equal(m.needle.shortName, 'model'); + assert.equal(m.worstStatus, 'ok'); +}); + +test('ccleft source: model-specific weekly arrives as kind weekly with id seven_day_<model>', () => { + const m = model(resp({ limits: [ + { id: 'seven_day', kind: 'weekly', percent_used: 20, pct_remaining: 80, resets_at: '2026-11-16T07:00:00Z', duration_mins: 10080 }, + { id: 'seven_day_opus', kind: 'weekly', percent_used: 85, pct_remaining: 15, resets_at: '2026-11-16T07:00:00Z', duration_mins: 10080, scope: { scope: 'opus', binding: 'false' } }, + { id: 'five_hour', kind: 'five_hour', percent_used: 10, pct_remaining: 90, resets_at: '2026-11-13T23:00:00Z', duration_mins: 300 }, + ] }), NOW); + assert.deepEqual(plain(m.dials.map(d => d.label)), ['Weekly', 'Weekly (opus)', '5-hour']); + assert.equal(m.needle.shortName, 'model'); + assert.equal(m.needle.pctLeft, 15); + assert.equal(m.worstStatus, 'warn'); + const bare = model(resp({ limits: [{ id: 'seven_day_scoped', kind: 'weekly', pct_remaining: 40, duration_mins: 10080, scope: { scope: 'scoped', binding: 'false' } }] }), NOW); + assert.equal(bare.dials[0].label, 'Weekly (model-specific)'); +}); + +test('credit pools and balances are not drawn as plan limits', () => { + const m = model(resp({ limits: [ + limit('seven_day', 'weekly', 60, 10080), + { id: 'extra_usage', kind: 'credits', pct_remaining: 5, duration_mins: 0 }, + ] }), NOW); + assert.equal(m.dials.length, 1); + assert.equal(m.needle.pctLeft, 60); +}); diff --git a/src/pkg/dashboard/subscription_gauge_chip_test.go b/src/pkg/dashboard/subscription_gauge_chip_test.go new file mode 100644 index 0000000000..137995c43c --- /dev/null +++ b/src/pkg/dashboard/subscription_gauge_chip_test.go @@ -0,0 +1,55 @@ +package dashboard + +import ( + "strings" + "testing" +) + +// TestSubscriptionGaugeChipWiring pins the opt-in header chip: hidden by +// default, shown only for the owner with dashboard.subscription_gauge_header +// on, fed by the shared loader (no fetch of its own), and opening the +// Overview gauge on click. +func TestSubscriptionGaugeChipWiring(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + `id="oc-subscription-gauge"`, + `class="nav-chip nav-chip--status subscription-gauge-chip"`, + `data-action="openSubscriptionGaugeOverview"`, + `.subscription-gauge-chip[hidden] { display: none; }`, + `#oc-subscription-gauge .nav-chip__label,`, + } { + if !strings.Contains(html, want) { + t.Fatalf("header usage chip missing %q", want) + } + } + i := strings.Index(html, `id="oc-subscription-gauge"`) + tagStart := strings.LastIndex(html[:i], "<button") + tagEnd := strings.Index(html[i:], ">") + i + if tag := html[tagStart:tagEnd]; !strings.Contains(tag, " hidden") { + t.Fatalf("header usage chip must be hidden by default: %s", tag) + } + + chip := jsFunc(t, html, "renderSubscriptionGaugeChip") + for _, want := range []string{"subscriptionGaugeOwner()", "subscriptionGaugeHeaderEnabled()", "el.hidden = true", "subscriptionGaugeDialSvg(", "compact: true", "Claude: no reading"} { + if !strings.Contains(chip, want) { + t.Fatalf("renderSubscriptionGaugeChip missing %q:\n%s", want, chip) + } + } + if strings.Contains(chip, "fetch(") || strings.Contains(chip, "innerHTML") { + t.Fatalf("the chip must reuse the shared readings and build DOM without innerHTML:\n%s", chip) + } + header := jsFunc(t, html, "subscriptionGaugeHeaderEnabled") + if !strings.Contains(header, "dashboardFeatureEnabled('subscription_gauge_header')") { + t.Fatalf("the chip must follow dashboard.subscription_gauge_header:\n%s", header) + } + surfaces := jsFunc(t, html, "renderSubscriptionGaugeSurfaces") + if !strings.Contains(surfaces, "renderSubscriptionGaugeChip()") { + t.Fatalf("shared render must update the chip:\n%s", surfaces) + } + open := jsFunc(t, html, "openSubscriptionGaugeOverview") + for _, want := range []string{"ocNavigate('overview-section')", "overview-usage-gauge", "scrollIntoView"} { + if !strings.Contains(open, want) { + t.Fatalf("openSubscriptionGaugeOverview missing %q:\n%s", want, open) + } + } +} diff --git a/src/pkg/dashboard/subscription_gauge_overview_test.go b/src/pkg/dashboard/subscription_gauge_overview_test.go new file mode 100644 index 0000000000..eabb6160b2 --- /dev/null +++ b/src/pkg/dashboard/subscription_gauge_overview_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "strings" + "testing" +) + +// TestSubscriptionGaugeOverviewWiring pins the Overview fuel gauge: its host +// sits outside the re-rendered overview card, it is on by default with a +// per-browser hide checkbox for the owner, and it never fetches the +// owner-only headroom endpoint for anyone else. +func TestSubscriptionGaugeOverviewWiring(t *testing.T) { + html := indexHTML(t) + host := `<div class="overview-usage-gauge" id="overview-usage-gauge" role="region" aria-label="Claude subscription usage" hidden></div>` + card := `<div class="overview-card" id="overview-card"></div>` + hi, ci := strings.Index(html, host), strings.Index(html, card) + if hi < 0 || ci < 0 || hi > ci { + t.Fatalf("Overview gauge host must exist, hidden by default, before (and outside) #overview-card: host=%d card=%d", hi, ci) + } + + defaults := jsFunc(t, html, "overviewDefaultChartState") + if !strings.Contains(defaults, "showUsageGauge: true") { + t.Fatalf("Overview gauge must be on by default:\n%s", defaults) + } + state := jsFunc(t, html, "overviewChartsState") + if !strings.Contains(state, "showUsageGauge: saved.showUsageGauge !== false") { + t.Fatalf("Overview gauge preference must default to on when unset:\n%s", state) + } + setter := jsFunc(t, html, "setOverviewUsageGauge") + for _, want := range []string{"state.showUsageGauge = !!checked", "subscriptionGaugeRefresh()"} { + if !strings.Contains(setter, want) { + t.Fatalf("setOverviewUsageGauge missing %q:\n%s", want, setter) + } + } + popover := jsFunc(t, html, "renderOverviewSettingsPopover") + if !strings.Contains(popover, `subscriptionGaugeOwner()`) || !strings.Contains(popover, `data-change-action="setOverviewUsageGauge"`) || !strings.Contains(popover, "Show Claude usage gauge") { + t.Fatalf("Overview settings must offer an owner-only Show Claude usage gauge checkbox:\n%s", popover) + } + + wanted := jsFunc(t, html, "subscriptionGaugeWanted") + if !strings.Contains(wanted, "subscriptionGaugeOwner() &&") { + t.Fatalf("the gauge must be wanted only for the owner:\n%s", wanted) + } + owner := jsFunc(t, html, "subscriptionGaugeOwner") + if !strings.Contains(owner, "dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')") { + t.Fatalf("owner check changed:\n%s", owner) + } + fetchBody := jsFunctionBody(t, html, "async function subscriptionGaugeFetch(") + gate := strings.Index(fetchBody, "if (!subscriptionGaugeWanted()) return;") + call := strings.Index(fetchBody, "fetch('/api/providers/headroom')") + if gate < 0 || call < 0 || gate > call { + t.Fatalf("the headroom fetch must come after the owner gate:\n%s", fetchBody) + } + if n := strings.Count(html, "fetch('/api/providers/headroom')"); n != 2 { + // One in the gauge loader, one in the existing rotation table loader. + t.Fatalf("expected exactly 2 headroom fetch sites (gauge + rotation table), found %d", n) + } + + overview := jsFunc(t, html, "renderSubscriptionGaugeOverview") + for _, want := range []string{"subscriptionGaugeOwner()", "subscriptionGaugeOverviewEnabled()", "m.usesClaude", "textContent", "host.hidden = true"} { + if !strings.Contains(overview, want) { + t.Fatalf("renderSubscriptionGaugeOverview missing %q:\n%s", want, overview) + } + } + if strings.Contains(overview, "innerHTML") { + t.Fatalf("Overview gauge must build its DOM without innerHTML:\n%s", overview) + } + dial := jsFunc(t, html, "subscriptionGaugeDialSvg") + if !strings.Contains(dial, "createElementNS") || strings.Contains(dial, "innerHTML") { + t.Fatalf("fuel dial must be built with createElementNS, not innerHTML:\n%s", dial) + } + + render := jsFunc(t, html, "render") + if !strings.Contains(render, "subscriptionGaugeRefresh()") { + t.Fatalf("render() must re-evaluate the gauge on every status frame") + } + auth := jsFunc(t, html, "showGHAuthState") + if !strings.Contains(auth, "subscriptionGaugeRefresh()") { + t.Fatalf("showGHAuthState must re-evaluate the gauge once the role resolves") + } +} diff --git a/src/pkg/rotation/rotation.go b/src/pkg/rotation/rotation.go index 61ff519e52..4da6b147af 100644 --- a/src/pkg/rotation/rotation.go +++ b/src/pkg/rotation/rotation.go @@ -256,6 +256,8 @@ func sanitizeProbeError(msg string) string { } // MarshalJSON includes the probe error text alongside the exported fields. +// PlanType is emitted only when a probe reported one, so readings without a +// plan (Claude's today) serialize exactly as before. func (h Headroom) MarshalJSON() ([]byte, error) { type alias struct { Provider string `json:"provider"` @@ -268,6 +270,7 @@ func (h Headroom) MarshalJSON() ([]byte, error) { ProbeErr string `json:"probe_error,omitempty"` Error string `json:"error,omitempty"` ProbeErrCause string `json:"probe_error_cause,omitempty"` + PlanType string `json:"plan_type,omitempty"` } return json.Marshal(alias{ Provider: h.Provider, @@ -280,6 +283,7 @@ func (h Headroom) MarshalJSON() ([]byte, error) { ProbeErr: h.ProbeError(), Error: h.ProbeError(), ProbeErrCause: string(h.ProbeErrCause), + PlanType: h.PlanType, }) } diff --git a/src/pkg/rotation/rotation_test.go b/src/pkg/rotation/rotation_test.go index cb93ee6448..a64a2093eb 100644 --- a/src/pkg/rotation/rotation_test.go +++ b/src/pkg/rotation/rotation_test.go @@ -2,10 +2,13 @@ package rotation import ( "context" + "encoding/json" "errors" "net/http" "net/http/httptest" + "strings" "testing" + "time" "github.com/hivecommons/hive/pkg/config" ) @@ -235,3 +238,36 @@ func TestManager_UnknownBackend(t *testing.T) { t.Error("Exhausted = true for unmapped backend, want false") } } + +// TestHeadroomMarshalJSONPlanType checks plan_type is emitted only when a +// probe reported one, and that a reading without one serializes exactly as +// it did before the field existed. +func TestHeadroomMarshalJSONPlanType(t *testing.T) { + at := time.Date(2026, 11, 13, 17, 0, 0, 0, time.UTC) + reset := time.Date(2026, 11, 20, 2, 0, 0, 0, time.UTC) + h := Headroom{ + Provider: "anthropic", + Available: true, + PctRemaining: 62, + ResetAt: reset, + CapturedAt: at, + Limits: []LimitWindow{{ID: "weekly_all", Kind: "weekly", PercentUsed: 38, PctRemaining: 62, ResetAt: reset, DurationMins: 10080}}, + } + data, err := json.Marshal(h) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + want := `{"provider":"anthropic","available":true,"pct_remaining":62,"reset_at":"2026-11-20T02:00:00Z","limits":[{"id":"weekly_all","kind":"weekly","percent_used":38,"pct_remaining":62,"resets_at":"2026-11-20T02:00:00Z","duration_mins":10080}],"captured_at":"2026-11-13T17:00:00Z"}` + if string(data) != want { + t.Fatalf("reading without a plan changed shape:\n got %s\nwant %s", data, want) + } + + h.PlanType = "pro" + data, err = json.Marshal(h) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if !strings.Contains(string(data), `"plan_type":"pro"`) { + t.Fatalf("plan_type missing from reading with a plan: %s", data) + } +} From d5e484c6bb482cd709fdc4be360bfa7a67ca02b6 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 14:38:22 -0500 Subject: [PATCH 168/223] =?UTF-8?q?=F0=9F=90=9B=20fix(inference):=20trust?= =?UTF-8?q?=20the=20hive's=20own=20UID=20on=20the=20translator=20while=20p?= =?UTF-8?q?rivate=20forwarding=20is=20off=20(#18090)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(inference): trust the hive's own UID on the translator while private forwarding is off On v5 pods the hive drops to dev (UID 1001), which is also the UID map's ProxyUID, and every agent without a per-agent UID runs as it. #14505 narrowed the translator's internal-caller exemption to root only, so those agents' inference requests were refused as uid_not_mapped (#18061). The proxy UID was refused because the plain-forward and uninspected tunnel legs could relay an agent-controlled request to :18444 from the proxy's socket (#14448). Those legs dial only globally routable addresses unless HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true, so accept the proxy UID's header name only while that guard is in place, and log reason=proxy_uid_private_forward when it is refused. Hive-Run: hivecommons/hive#18061 Hive-Plan: issue-18061 translator proxy-UID identity Hive-Spec: issue-18061#proxy-uid-identity Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ♻️ refactor(proxy): move internalTranslatorUID beside the refusal reasons github_proxy_inference.go reached 801 lines, over the crux 800-line limit for non-test Go files. Move the new helper into inference_refusal.go, next to the proxy_uid_private_forward reason it mirrors. No behaviour change. Hive-Run: hivecommons/hive#18061 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 📖 docs(proxy): note the translator's proxy-UID refusal under private forwarding Applies the docs-currency review suggestion: env-vars.md now says that HIVE_PROXY_ALLOW_PRIVATE_FORWARD also makes the translator refuse the proxy UID (reason=proxy_uid_private_forward). Also pins the P3 follow-up that root stays trusted while private forwarding is allowed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18061-translator-proxy-uid.md | 1 + src/crux/proxy/github_proxy_inference.go | 14 +++++-- src/crux/proxy/inference_refusal.go | 15 ++++++- .../inference_translator_identity_test.go | 39 ++++++++++++++++++- src/docs/env-vars.md | 2 +- 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 changelog.d/fixed-18061-translator-proxy-uid.md diff --git a/changelog.d/fixed-18061-translator-proxy-uid.md b/changelog.d/fixed-18061-translator-proxy-uid.md new file mode 100644 index 0000000000..a4d62970b6 --- /dev/null +++ b/changelog.d/fixed-18061-translator-proxy-uid.md @@ -0,0 +1 @@ +- Stop refusing inference requests from the hive's own UID (`dev`, 1001 on v5 pods) when private plain-HTTP forwarding is off, so agents without a per-agent UID reach their inference route again; the proxy UID is still refused when `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` (#18061). diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 154abf41f1..4d7333ded7 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -49,8 +49,16 @@ const inferenceAPIKeyPrefix = "sk-hive-" // - a UID mapped to an agent is authoritative; a header naming someone else // is logged and overridden; // - the root control plane keeps the header name — the hive is the party -// that issues these keys; the proxy UID cannot assert a name because its -// upstream sockets can relay agent-controlled requests; +// that issues these keys; +// - the proxy UID keeps the header name only while the private-forward +// guard is in place. On v5 pods the hive drops to `dev`, which IS the +// proxy UID, and every agent without its own UID runs as it, so refusing +// that UID outright refused the hive's own agents (#18061). The reason it +// was refused (#14448) is that the proxy's plain-forward and uninspected +// tunnel legs can relay an agent-controlled request to this loopback +// port from the proxy's own socket; those legs dial only globally +// routable addresses unless HIVE_PROXY_ALLOW_PRIVATE_FORWARD lifts the +// guard, so with the guard lifted the proxy UID is still refused; // - an unmapped UID, or no UID map at all, accepts the header only under // HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path // honours for deployments without per-agent UID separation. @@ -74,7 +82,7 @@ func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { } return name, true } - if uid == 0 && p.uidMap.IsInternalUID(uid) { + if p.internalTranslatorUID(uid) { return claimed, true } } diff --git a/src/crux/proxy/inference_refusal.go b/src/crux/proxy/inference_refusal.go index 922db429e8..e797f17cd1 100644 --- a/src/crux/proxy/inference_refusal.go +++ b/src/crux/proxy/inference_refusal.go @@ -2,9 +2,20 @@ package proxy import "net/http" +// internalTranslatorUID reports whether uid is the hive's own control plane +// for the translator's purposes: root always, and the proxy UID only when the +// proxy cannot relay to private addresses (see translatorAgentName). +func (p *GitHubProxy) internalTranslatorUID(uid int) bool { + if !p.uidMap.IsInternalUID(uid) { + return false + } + return uid == 0 || !p.allowPrivateForward +} + // translatorRefusalAttrs are the log attributes for a refused translator // request. Beyond remote/method/path they say why identification failed (no UID -// map, socket not resolvable, or UID not mapped to an agent), the socket owner's +// map, socket not resolvable, proxy UID refused because private forwarding is +// enabled, or UID not mapped to an agent), the socket owner's // UID when known, and whether an x-api-key was sent, so the refusal is // actionable without weakening it. The key value is never logged. func (p *GitHubProxy) translatorRefusalAttrs(r *http.Request) []any { @@ -16,6 +27,8 @@ func (p *GitHubProxy) translatorRefusalAttrs(r *http.Request) []any { uid, err := uidForRemoteAddr(r.RemoteAddr) if err != nil { attrs = append(attrs, "reason", "socket_owner_unresolved", "error", err.Error()) + } else if p.uidMap.IsInternalUID(uid) && p.allowPrivateForward { + attrs = append(attrs, "reason", "proxy_uid_private_forward", "uid", uid) } else { attrs = append(attrs, "reason", "uid_not_mapped", "uid", uid) } diff --git a/src/crux/proxy/inference_translator_identity_test.go b/src/crux/proxy/inference_translator_identity_test.go index 21cf3f4179..d34e3ea640 100644 --- a/src/crux/proxy/inference_translator_identity_test.go +++ b/src/crux/proxy/inference_translator_identity_test.go @@ -96,12 +96,49 @@ func TestTranslatorAgentName_UnmappedRefusedUnlessAdvisoryOK(t *testing.T) { func TestTranslatorAgentName_InternalUIDKeepsHeader(t *testing.T) { fakeProcNetTCPUID(t, 0) p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} - // Root is the control plane; the separate proxy UID is not trusted. + // Root is the control plane whatever the private-forward setting. p.uidMap = &agent.UIDMap{BaseUID: 2001, ProxyUID: 2003, Agents: map[string]int{"quality": 2002}} if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "supervisor" { t.Fatalf("internal uid: got (%q,%v), want (supervisor,true)", name, ok) } + // Lifting the private-forward guard refuses only the proxy UID, never root. + p.allowPrivateForward = true + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "supervisor" { + t.Fatalf("root, private forward allowed: got (%q,%v), want (supervisor,true)", name, ok) + } +} + +// TestTranslatorAgentName_ProxyUIDTrustedOnlyWhileForwardGuarded pins #18061: +// on v5 pods the hive drops to the proxy UID (dev, 1001) and agents without a +// per-agent UID run as it, so the proxy UID keeps its header name — but only +// while the private-forward guard stops the proxy relaying agent requests to +// this loopback port (#14448). With the guard lifted it is refused again. +func TestTranslatorAgentName_ProxyUIDTrustedOnlyWhileForwardGuarded(t *testing.T) { + fakeProcNetTCP(t) + var buf strings.Builder + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(&buf, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, ProxyUID: 2003, Agents: map[string]int{"quality": 2002}} + + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "supervisor" { + t.Fatalf("proxy uid, forward guarded: got (%q,%v), want (supervisor,true)", name, ok) + } + + p.allowPrivateForward = true + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); ok || name != "" { + t.Fatalf("proxy uid, private forward allowed: got (%q,%v), want refused", name, ok) + } + p.inferenceTranslatorHandler().ServeHTTP(httptest.NewRecorder(), translatorReq("127.0.0.1:43210", "supervisor")) + if !strings.Contains(buf.String(), "reason=proxy_uid_private_forward") { + t.Errorf("refusal log %q lacks reason=proxy_uid_private_forward", buf.String()) + } + + // An agent allocated the proxy UID is never internal: its own name wins. + p.allowPrivateForward = false + p.uidMap = &agent.UIDMap{BaseUID: 2001, ProxyUID: 2003, Agents: map[string]int{"scanner": 2003}} + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "scanner" { + t.Fatalf("agent at proxy uid: got (%q,%v), want (scanner,true)", name, ok) + } } // TestTranslatorAgentName_NoUIDMap pins the no-map deployments: the header is diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index e9e8264b48..6b846fe6ac 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -221,7 +221,7 @@ new value at the same time. | `HIVE_STATIC_DIR` | No | `/opt/hive/proxy/public` | Static asset directory for the Node proxy. | | `HIVE_PROXY_EGRESS_MARK` | No | `0x1112` | Packet mark exempted from the MITM egress redirect. | | `HIVE_PROXY_ADVISORY_OK` | No | `false` | Allows the spoke to start when the forced-proxy egress redirect cannot be installed (no `CAP_NET_ADMIN`/iptables). Enforcement becomes advisory-only — agents can bypass the proxy. Also gates whether the Go proxy trusts a self-asserted `Proxy-Authorization` header as agent identity when its UID map is unavailable (N7, #3841) — off by default, an unidentified caller is treated as `ADVISORY` (writes blocked) rather than whatever name it claims. See [security-model.md](security-model.md#5-network-and-credentials). | -| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the proxy's destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082), [#16785](https://github.com/hivecommons/hive/pull/16785)). Off by default: a plain-HTTP (non-CONNECT) forward, a CONNECT to a host the proxy does not inspect, and a transparent-TLS passthrough are each dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, and it lifts the guard for all three legs at once, for deployments that genuinely egress to a private network through the proxy (a LAN mirror on a native install); the proxy logs a boot-time warning when it is set. Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). | +| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the proxy's destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082), [#16785](https://github.com/hivecommons/hive/pull/16785)). Off by default: a plain-HTTP (non-CONNECT) forward, a CONNECT to a host the proxy does not inspect, and a transparent-TLS passthrough are each dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, and it lifts the guard for all three legs at once, for deployments that genuinely egress to a private network through the proxy (a LAN mirror on a native install); the proxy logs a boot-time warning when it is set. Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). With the guard lifted, the inference translator also stops trusting the proxy UID (`dev`, 1001 on v5 pods): agents that run as the hive user without a per-agent UID are refused with `reason=proxy_uid_private_forward` ([#18061](https://github.com/hivecommons/hive/issues/18061), [#14448](https://github.com/hivecommons/hive/issues/14448)). | | `HIVE_PROXY_INJECT_GH_AUTH` | No | unset (off on every hive, including hosted) | Exact `true` enables proxy-side App-token injection. **Requires GitHub App auth. Refuses startup (exit 19) with `HIVE_PROXY_ADVISORY_OK=true`.** Other spellings do not enable it. Read [behavior and exceptions](#proxy-credential-injection). | | `HIVE_DEPLOYMENT_RUNTIME` | No | none (config `deployment.runtime` fallback; hub-registered Kubernetes hives are auto-detected) | Declares the deployment runtime (`kubernetes`, `podman-quadlet`, or `docker-compose`) so the dashboard can offer standalone upgrades. Unset/unrecognized means `unknown` and the upgrade button stays hidden. See [dashboard-standalone-upgrades.md](dashboard-standalone-upgrades.md). | | `HIVE_DEPLOYMENT_PODMAN_MODE` | Only when runtime is `podman-quadlet` | none (config `deployment.podman_mode` fallback) | `rootless` (`systemctl --user`) or `rootful` (system manager). Without an explicit mode the Podman/Quadlet upgrade path stays disabled. | From 2cc6995160f1c1b0af9471292b0a7b6b77f52ee8 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 14:45:03 -0500 Subject: [PATCH 169/223] =?UTF-8?q?=F0=9F=90=9B=20automerge:=20contributor?= =?UTF-8?q?=20lane=20treats=20incomplete=20file=20list=20as=20a=20standing?= =?UTF-8?q?=20skip=20(#18124)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PR with more changed files than GitHub's files API returns (3000) can never have its diff fences read, but the contributor lane returned ErrIncompletePRFileList as an error, so every sweep WARNed "contributor-lane automerge sweep skipped PR" and the fleet reporter filed it as unclassified_error_repeated (117 in 10m on one PR). Return files-evidence with a nil error, as adoptionDiffRefusal already does: the PR stays failed closed, the skip is logged as an ordinary "automerge skip", and the relay gate returns a refusal instead of an evidence error. Transport failures still return the error. Fixes #18121 Hive-Run: hivecommons/hive#18121 Hive-Plan: selfreport unclassified_error_repeated triage Hive-Spec: contributor-lane#files-evidence Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18121-contributor-lane-incomplete-files.md | 1 + src/pkg/github/automerge/automerge_contributor_lane.go | 7 +++++++ .../github/automerge/automerge_contributor_lane_test.go | 7 +++++++ src/pkg/github/automerge/automerge_relay_gate_test.go | 9 +++++++++ 4 files changed, 24 insertions(+) create mode 100644 changelog.d/fixed-18121-contributor-lane-incomplete-files.md diff --git a/changelog.d/fixed-18121-contributor-lane-incomplete-files.md b/changelog.d/fixed-18121-contributor-lane-incomplete-files.md new file mode 100644 index 0000000000..a823e5198f --- /dev/null +++ b/changelog.d/fixed-18121-contributor-lane-incomplete-files.md @@ -0,0 +1 @@ +- The contributor-lane automerge sweep now treats a PR with more changed files than GitHub's files API returns (3000) as a standing `files-evidence` skip instead of re-warning every sweep. diff --git a/src/pkg/github/automerge/automerge_contributor_lane.go b/src/pkg/github/automerge/automerge_contributor_lane.go index 72ee6aa15a..39c13f7048 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane.go +++ b/src/pkg/github/automerge/automerge_contributor_lane.go @@ -1405,6 +1405,13 @@ func (c *Engine) evaluateContributorLanePR(ctx context.Context, displayRepo, own c.info("contributor-lane admitted", "repo", displayRepo, "pr", number, "author", author, "reason", "contributor-lane:auto-promoted", "merged_prs", claim.AutoPromoted, "scope", contributorAutoPromotionScopeName(claim.AutoPromotedHiveWide), "repos", claim.AutoPromotedRepos) } files, err := ListChangedFiles(caller, c.gh, owner, repo, pr) + if errors.Is(err, hgithub.ErrIncompletePRFileList) { + // More changed files than GitHub's files API will ever return: the + // diff fences cannot be read on any tick, so this is a standing + // verdict (fail closed, nil error) like adoptionDiffRefusal's, not a + // transient fault re-WARNed every sweep (hivecommons/hive#18121). + return eval, contributorLaneReasonFilesEvidence, nil + } if err != nil { return eval, contributorLaneReasonFilesEvidence, err } diff --git a/src/pkg/github/automerge/automerge_contributor_lane_test.go b/src/pkg/github/automerge/automerge_contributor_lane_test.go index 1a64bdf960..e54e5d9c31 100644 --- a/src/pkg/github/automerge/automerge_contributor_lane_test.go +++ b/src/pkg/github/automerge/automerge_contributor_lane_test.go @@ -60,6 +60,7 @@ type contributorLaneFixture struct { omitAssociation bool files []map[string]any filesStatus int + changedFiles int // changed_files on the PR payload; 0 omits it listStatus int getStatus int recheckHeadSHA string @@ -219,6 +220,9 @@ func newContributorLaneAPI(t *testing.T, fx *contributorLaneFixture) *httptest.S if !fx.omitAssociation { p["author_association"] = association } + if fx.changedFiles != 0 { + p["changed_files"] = fx.changedFiles + } if fx.headCommittedAt != "" { p["updated_at"] = fx.headCommittedAt } @@ -760,6 +764,9 @@ func TestTrySweepContributorLanePRSkipReasons(t *testing.T) { wantMerges: 1, }, {name: "files fetch error", mutate: func(fx *contributorLaneFixture) { fx.filesStatus = http.StatusInternalServerError }, wantReason: contributorLaneReasonFilesEvidence, wantErr: true}, + // More changed files than the files API returns is a standing + // verdict, not a transient error re-WARNed every sweep (#18121). + {name: "incomplete files list", mutate: func(fx *contributorLaneFixture) { fx.changedFiles = 5204 }, wantReason: contributorLaneReasonFilesEvidence}, {name: "claim comments fetch error", mutate: func(fx *contributorLaneFixture) { fx.issueCommentsSt = http.StatusInternalServerError }, wantReason: "claim-comments-fetch", wantErr: true}, { name: "reporter-trust hold", diff --git a/src/pkg/github/automerge/automerge_relay_gate_test.go b/src/pkg/github/automerge/automerge_relay_gate_test.go index 0edbc193bd..cb585124ee 100644 --- a/src/pkg/github/automerge/automerge_relay_gate_test.go +++ b/src/pkg/github/automerge/automerge_relay_gate_test.go @@ -251,6 +251,15 @@ func TestRelayMergeRefusalFailsClosed(t *testing.T) { t.Fatalf("err = %v, want lane evidence error", err) } }) + t.Run("incomplete files list refuses", func(t *testing.T) { + fx := validContributorFixture() + fx.changedFiles = 5204 + engine, _ := relayGateEngine(t, &fx, defaultContributorLanePolicy(), Options{}) + refusal, err := engine.RelayMergeRefusal(context.Background(), "widget", 7, "sha7") + if err != nil || !strings.Contains(refusal, contributorLaneReasonFilesEvidence) { + t.Fatalf("refusal=%q err=%v, want standing files-evidence refusal", refusal, err) + } + }) } func TestRelayMergeRefusalText(t *testing.T) { From c0d13d934d6e7c18685b3e6579af27cb47f2935a Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 19:59:36 +0000 Subject: [PATCH 170/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.246.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.246.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 11 +++++++++++ changelog.d/added-subscription-usage-gauge.md | 1 - .../fixed-18084-close-on-merge-unreadable-pr.md | 1 - .../fixed-quota-exhausted-agents-not-kicked.md | 1 - 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/added-subscription-usage-gauge.md delete mode 100644 changelog.d/fixed-18084-close-on-merge-unreadable-pr.md delete mode 100644 changelog.d/fixed-quota-exhausted-agents-not-kicked.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ad82bdc8..0747a9d687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.246.0) + +### Added + +- Dashboard: an owner-only fuel gauge on the Overview now shows how much of the hive's Claude plan usage is left in each limit (weekly, plus a 5-hour or model-specific limit when the plan has one) and when each resets, turning amber at 25% left and red at 10%. It is on by default whenever the hive uses Claude, can be hidden from the Overview settings, and can also be shown as a header chip on every page with `dashboard.subscription_gauge_header`. It displays the usage readings hive already makes, so it adds no calls to Claude and changes nothing about measurement or provider rotation; `/api/providers/headroom` readings now also carry `plan_type` when a provider reports one. + +### Fixed + +- Close-on-merge no longer re-warns every tick for merged PRs it cannot read (404/403); it logs once per recheck window. +- **Agents out of provider quota are no longer kicked every cycle** ([#18112](https://github.com/hivecommons/hive/issues/18112)). The provider spending-limit gate only saw agents behind the hive's inference gateway, so an agent whose own provider refused it — GitHub Copilot's "You have exceeded your monthly quota" — kept being kicked and sent review work all night, each turn ending at once, until its review dispatches expired and pull requests were parked for a human. The governor now holds kicks per agent while that agent's `QuotaExhausted` flag is set, lets one probe kick through per `governor.provider_budget.probe_interval_s` (default 30 minutes) to notice when the quota resets, and resumes normal kicking as soon as the flag clears. Other agents are never held, and the "provider quota exhausted" alert now says the kicks are held and how often they are retried. + ## 2026-11-13 (v5.245.0) ### Added diff --git a/changelog.d/added-subscription-usage-gauge.md b/changelog.d/added-subscription-usage-gauge.md deleted file mode 100644 index 87e70297fb..0000000000 --- a/changelog.d/added-subscription-usage-gauge.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: an owner-only fuel gauge on the Overview now shows how much of the hive's Claude plan usage is left in each limit (weekly, plus a 5-hour or model-specific limit when the plan has one) and when each resets, turning amber at 25% left and red at 10%. It is on by default whenever the hive uses Claude, can be hidden from the Overview settings, and can also be shown as a header chip on every page with `dashboard.subscription_gauge_header`. It displays the usage readings hive already makes, so it adds no calls to Claude and changes nothing about measurement or provider rotation; `/api/providers/headroom` readings now also carry `plan_type` when a provider reports one. diff --git a/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md b/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md deleted file mode 100644 index 76d3978235..0000000000 --- a/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md +++ /dev/null @@ -1 +0,0 @@ -- Close-on-merge no longer re-warns every tick for merged PRs it cannot read (404/403); it logs once per recheck window. diff --git a/changelog.d/fixed-quota-exhausted-agents-not-kicked.md b/changelog.d/fixed-quota-exhausted-agents-not-kicked.md deleted file mode 100644 index cce7aae2e1..0000000000 --- a/changelog.d/fixed-quota-exhausted-agents-not-kicked.md +++ /dev/null @@ -1 +0,0 @@ -- **Agents out of provider quota are no longer kicked every cycle** ([#18112](https://github.com/hivecommons/hive/issues/18112)). The provider spending-limit gate only saw agents behind the hive's inference gateway, so an agent whose own provider refused it — GitHub Copilot's "You have exceeded your monthly quota" — kept being kicked and sent review work all night, each turn ending at once, until its review dispatches expired and pull requests were parked for a human. The governor now holds kicks per agent while that agent's `QuotaExhausted` flag is set, lets one probe kick through per `governor.provider_budget.probe_interval_s` (default 30 minutes) to notice when the quota resets, and resumes normal kicking as soon as the flag clears. Other agents are never held, and the "provider quota exhausted" alert now says the kicks are held and how often they are retried. From 945313d341010334ce4b00f0c82d6cad2f112426 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 15:12:26 -0500 Subject: [PATCH 171/223] =?UTF-8?q?=E2=9C=A8=20knowledge:=20hot-load=20con?= =?UTF-8?q?nectors=20saved=20from=20the=20dashboard=20(no=20hive=20restart?= =?UTF-8?q?)=20(#18099)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: hot-load connectors saved from the dashboard (no hive restart) The connector syncer was built once at boot and stayed nil when the hive started with no knowledge.connectors, so a connector added from the Knowledge Integrations dashboard was saved to config but every Check, Sync and Connect call answered "no knowledge connectors are configured" until a restart. - connector.Syncer gains Reconfigure: adds new names, rebuilds entries whose config changed (keeping cursor and counts, as across a restart), removes dropped names, and starts/stops their Run loops while it runs. Unchanged connectors keep their connector and loop. Entries and order are now guarded by the syncer mutex. - The dashboard PUT of knowledge.connectors calls a new Dependencies.ReloadKnowledgeConnectors hook and swaps the runtime under a lock; every read goes through Server.knowledgeConnectors(). - boot wires the hook: build and start the syncer from the boot inputs when none is running, else reconfigure it in place. - The knowledge.publish mirror is unchanged and still needs a restart; the docs say so. Hive-Run: hivecommons/hive#18091 Hive-Plan: issue-18091 hot-load knowledge connector syncer Hive-Spec: knowledge-connectors#hot-load Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🐛 knowledge: wait on loop exit instead of fixed sleeps in reconfigure test TestSleepRatchet failed (197 time.Sleep calls vs baseline 195): the two sleeps added in TestSyncerReconfigureWhileRunning. Disable the connector on its own and wait for the syncer's loop WaitGroup to drain, which proves its loop stopped, before adding the second connector. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 📖 docs: re-anchor api-reference server.go citations after #18091 shift The connector hot-load wiring moves the server.go route table, so 16 api-reference citations pointed at stale lines (failing changelog-fragment-guard, the relative-links check and, through them, native-queue-policy). Rewritten with check-api-reference-citations.sh --fix on top of v5 @ a68f5c89e. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-18091-connector-hotload.md | 1 + src/cmd/hive/boot.go | 21 +- src/cmd/hive/boot_knowledge_connectors.go | 34 +++ .../boot_knowledge_connectors_reload_test.go | 52 +++++ src/cmd/hive/main.go | 25 +-- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/knowledge.md | 6 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/knowledge-connectors.md | 5 +- .../api_config_knowledge_connectors.go | 9 +- src/pkg/dashboard/api_knowledge_connectors.go | 25 +++ .../api_knowledge_connectors_hotload_test.go | 121 +++++++++++ src/pkg/dashboard/deps.go | 10 +- src/pkg/dashboard/server.go | 4 + src/pkg/knowledge/connector/syncer.go | 204 ++++++++++++++---- .../connector/syncer_reconfigure_test.go | 157 ++++++++++++++ 18 files changed, 618 insertions(+), 88 deletions(-) create mode 100644 changelog.d/added-18091-connector-hotload.md create mode 100644 src/cmd/hive/boot_knowledge_connectors_reload_test.go create mode 100644 src/pkg/dashboard/api_knowledge_connectors_hotload_test.go create mode 100644 src/pkg/knowledge/connector/syncer_reconfigure_test.go diff --git a/changelog.d/added-18091-connector-hotload.md b/changelog.d/added-18091-connector-hotload.md new file mode 100644 index 0000000000..fa62426074 --- /dev/null +++ b/changelog.d/added-18091-connector-hotload.md @@ -0,0 +1 @@ +- Knowledge connectors added, edited, enabled, disabled or removed from the Knowledge Integrations dashboard now take effect without restarting the hive: a hive booted with no connectors builds and starts the syncer on the first save (Check, Sync and Connect stop answering `no knowledge connectors are configured`), and a running syncer is reconfigured in place so unchanged connectors keep syncing. The `knowledge.publish` mirror still needs a restart when its connector changes ([#18091](https://github.com/hivecommons/hive/issues/18091)). diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 99ed7f91c1..93a38730a6 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -216,14 +216,19 @@ type boot struct { saved *snapshot.PersistedState // bootKnowledge - gitSyncer *knowledge.GitSyncer - knowledgeConnectors *connector.Syncer - inboxSvc *inbox.Service - knowledgePublish *connector.Mirror - promotionScheduler *knowledge.PromotionScheduler - quotaAccount string - quotaPoolDir string - explicitQuotaPoolDir bool + gitSyncer *knowledge.GitSyncer + knowledgeConnectors *connector.Syncer + // knowledgeConnectorsMu guards knowledgeConnectors once the dashboard + // can hot-load it (#18091); startConnectorSyncer is the boot starter + // reused for a syncer first built after boot. + knowledgeConnectorsMu sync.Mutex + startConnectorSyncer func(ctx context.Context, s *connector.Syncer) + inboxSvc *inbox.Service + knowledgePublish *connector.Mirror + promotionScheduler *knowledge.PromotionScheduler + quotaAccount string + quotaPoolDir string + explicitQuotaPoolDir bool // knowledgeAPIFallback records that knowledgeAPI is the file-only API // bootKnowledge auto-enables when knowledge.enabled is false, not one // built from the configured layers/engine/curator (#9231). diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index 070f0f9b5e..de04a6344f 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -93,6 +93,7 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { // answers 200 with an empty list and the panel renders its empty state // (#17918) instead of waiting forever on a 404. b.bootInbox(knowledge.BaseDir()) + b.startConnectorSyncer = deps.startConnectorSyncer s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil { b.logger.Warn("knowledge connectors disabled", "error", err) @@ -101,17 +102,50 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { if s == nil { return } + b.knowledgeConnectorsMu.Lock() b.knowledgeConnectors = s + b.knowledgeConnectorsMu.Unlock() if deps.startConnectorSyncer != nil { deps.startConnectorSyncer(b.ctx, s) } b.logger.Info("knowledge connectors started", "count", len(b.cfg.Knowledge.Connectors)) } +// reloadKnowledgeConnectors applies a knowledge.connectors list saved from +// the dashboard to the running hive (#18091). With no syncer yet (the hive +// booted with zero connectors, or the boot build failed) it builds and +// starts one from the same inputs as bootKnowledgeConnectors; otherwise it +// reconfigures the running syncer in place, so unchanged connectors keep +// syncing undisturbed. The knowledge.publish mirror is not rebuilt: a +// changed publish connector still needs a restart. +func (b *boot) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) (dashboard.KnowledgeConnectorRuntime, error) { + b.knowledgeConnectorsMu.Lock() + defer b.knowledgeConnectorsMu.Unlock() + if b.knowledgeConnectors != nil { + if err := b.knowledgeConnectors.Reconfigure(connectorConfigs(entries)); err != nil { + return nil, err + } + b.logger.Info("knowledge connectors reloaded", "count", len(entries)) + return b.knowledgeConnectors, nil + } + s, err := newKnowledgeConnectorSyncer(entries, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) + if err != nil || s == nil { + return nil, err + } + b.knowledgeConnectors = s + if b.startConnectorSyncer != nil { + b.startConnectorSyncer(b.ctx, s) + } + b.logger.Info("knowledge connectors started", "count", len(entries), "hot_loaded", true) + return s, nil +} + // knowledgeConnectorRuntime exposes the syncer to the dashboard. A nil // *connector.Syncer must stay a nil interface so the dashboard reports // "no connectors configured" instead of calling methods on a nil pointer. func (b *boot) knowledgeConnectorRuntime() dashboard.KnowledgeConnectorRuntime { + b.knowledgeConnectorsMu.Lock() + defer b.knowledgeConnectorsMu.Unlock() if b.knowledgeConnectors == nil { return nil } diff --git a/src/cmd/hive/boot_knowledge_connectors_reload_test.go b/src/cmd/hive/boot_knowledge_connectors_reload_test.go new file mode 100644 index 0000000000..ae7e70aab6 --- /dev/null +++ b/src/cmd/hive/boot_knowledge_connectors_reload_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// A hive booted with zero connectors builds and starts the syncer the first +// time the dashboard saves one, then reconfigures that same syncer (#18091). +func TestReloadKnowledgeConnectors(t *testing.T) { + knowledge.SetBaseDirForTest(t, t.TempDir()) + b, log := newDepsTestBoot(t, &config.Config{}) + var started []*connector.Syncer + b.bootKnowledgeConnectors(bootKnowledgeDeps{startConnectorSyncer: func(_ context.Context, s *connector.Syncer) { + started = append(started, s) + }}) + if b.knowledgeConnectorRuntime() != nil || len(started) != 0 { + t.Fatal("no connectors at boot must leave the syncer off") + } + + if rt, err := b.reloadKnowledgeConnectors(nil); rt != nil || err != nil { + t.Fatalf("reload with none = %v, %v; want nil, nil", rt, err) + } + + docs := config.KnowledgeConnector{Name: "docs", Type: connector.TypeDocument, Layer: "project", + Scope: map[string]string{"url": "https://example.org/guide.html"}} + rt, err := b.reloadKnowledgeConnectors([]config.KnowledgeConnector{docs}) + if err != nil || rt == nil { + t.Fatalf("reload = %v, %v\n%s", rt, err, log.String()) + } + if len(started) != 1 || started[0] != b.knowledgeConnectors { + t.Fatalf("hot-loaded syncer not started once: %d", len(started)) + } + if sts := rt.Statuses(); len(sts) != 1 || sts[0].Name != "docs" { + t.Fatalf("statuses = %+v", sts) + } + + rt2, err := b.reloadKnowledgeConnectors(nil) + if err != nil || rt2 != rt { + t.Fatalf("second reload = %v, %v; want the same runtime", rt2, err) + } + if sts := rt2.Statuses(); len(sts) != 0 { + t.Fatalf("removed connector still listed: %+v", sts) + } + if len(started) != 1 { + t.Fatalf("reconfigure must not start a second syncer: %d", len(started)) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 4f5841fb67..008989016a 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1140,18 +1140,19 @@ func (b *boot) wireBootClosures() { // #3972: hand the ACMM advisor the SAME cached fleet-stats collector // the heartbeat reads, so its merge-success signal reuses the existing // 30-minute collect loop instead of issuing a second GitHub fetch. - FleetStats: b.fleetStatsCollector, - Activity: b.activityCollector, - RepoCost: b.repoCostCollector, - LaunchCost: b.launchCostCollector, - BeadSynthesizer: b.beadSynth, - KnowledgeConnectors: b.knowledgeConnectorRuntime(), - KnowledgePublish: b.knowledgePublishRuntime(), - Inbox: b.inboxSvc, - BeadStores: b.beadStores, - BeadStoreLoadFailures: b.beadStoreLoadFailures, - AuditLedger: auditLedger, - AuditJournal: auditJournal, + FleetStats: b.fleetStatsCollector, + Activity: b.activityCollector, + RepoCost: b.repoCostCollector, + LaunchCost: b.launchCostCollector, + BeadSynthesizer: b.beadSynth, + KnowledgeConnectors: b.knowledgeConnectorRuntime(), + ReloadKnowledgeConnectors: b.reloadKnowledgeConnectors, + KnowledgePublish: b.knowledgePublishRuntime(), + Inbox: b.inboxSvc, + BeadStores: b.beadStores, + BeadStoreLoadFailures: b.beadStoreLoadFailures, + AuditLedger: auditLedger, + AuditJournal: auditJournal, AuditPublisherFunc: func() convergenceaudit.FindingPublisher { return b.findingPublisher }, diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 08de2de8c9..dfd32421ab 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1712` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1713` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1714` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1716` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1717` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1718` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 41b4d16486..e301461e4e 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1719` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1723` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 5ac2c544e1..a1cb297651 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1692` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1693` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1694` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1700` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1705` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1706` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1707` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1708` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1709` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1710` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1711` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1696` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1697` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1698` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1704` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1709` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1710` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1711` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1712` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1713` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1714` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1715` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index a20c2621ba..08cafb1a1f 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -59,9 +59,9 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:442` | | `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:405` | | `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:406` | -| `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | -| `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | -| `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | +| `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:24` | +| `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:25` | +| `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:26` | | `GET` | `/api/knowledge/connectors/{name}/auth` | Dashboard auth/session | Interactive sign-in state for a connector (Plaud OAuth): connected, account, pending, auth_url; `?refresh=1` re-checks upstream (400 for connector types without interactive sign-in) | `pkg/dashboard/api_knowledge_connector_auth.go:25` | | `POST` | `/api/knowledge/connectors/{name}/auth/connect` | Owner only | Start the connector's sign-in flow (202 with state incl. `auth_url`) | `pkg/dashboard/api_knowledge_connector_auth.go:26` | | `POST` | `/api/knowledge/connectors/{name}/auth/disconnect` | Owner only | Sign out and revoke the connector's credential upstream | `pkg/dashboard/api_knowledge_connector_auth.go:27` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index e67c5dc458..8dcc35479a 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1782` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1786` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index 12dc561672..3f79c726d2 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -86,7 +86,7 @@ Status and **Sync now** need the connector syncer, which `hive` starts at boot w | `GET /api/knowledge/connectors` | dashboard session | `{"connectors": [...]}`: one status per configured connector, in config order | | `POST /api/knowledge/connectors/{name}/sync` | owner | Start a background sync now, even if the connector is disabled (`202` with the status; `404` unknown name; `409` if running) | -Saved changes persist to the config overlay; the syncer picks up added, edited or removed connectors on the next restart. +Saved changes persist to the config overlay and apply at once, without a restart: a hive booted with no connectors starts the syncer on the first save, and on a running syncer an added connector starts syncing, an edited one is rebuilt with its cursor and counts kept, and a disabled or removed one stops. Unchanged connectors keep syncing undisturbed. The `knowledge.publish` mirror is not rebuilt this way; a change to its connector still needs a restart. Each status object returned by `GET /api/knowledge/connectors` contains: @@ -269,7 +269,8 @@ if you do not want a sync to pull a new upstream release unreviewed. 1. The hive host (or container image) needs **Node.js ≥ 20** so `npx` can run the Plaud MCP server. 2. Add the connector from **Knowledge → 🔌 Integrations → Plaud voice notes** - (or the YAML above) and restart the hive so the syncer picks it up. + (or the YAML above). A dashboard save loads it at once; a YAML edit + needs a restart. 3. On the Plaud card click **Connect Plaud**. The card shows the Plaud authorization URL; open it, sign in and click **Authorize**. The OAuth callback lands on `localhost:8199` of the machine running hive, so for a diff --git a/src/pkg/dashboard/api_config_knowledge_connectors.go b/src/pkg/dashboard/api_config_knowledge_connectors.go index 68e2cb17ba..0471de50b4 100644 --- a/src/pkg/dashboard/api_config_knowledge_connectors.go +++ b/src/pkg/dashboard/api_config_knowledge_connectors.go @@ -264,6 +264,7 @@ func (s *Server) handleKnowledgeConnectorsPut(w http.ResponseWriter, r *http.Req if err := s.saveConfig(); err != nil { s.logger.Error("failed to persist config after knowledge connectors update", "error", err) } + s.reloadKnowledgeConnectors(next) names := make([]string, 0, len(next)) for _, c := range next { names = append(names, c.Name) @@ -292,8 +293,8 @@ func knowledgeConnectorPill(enabled bool, st *connector.Status) string { func (s *Server) knowledgeConnectorStatusRows() []knowledgeConnectorStatusRow { byName := map[string]connector.Status{} - if s.deps.KnowledgeConnectors != nil { - for _, st := range s.deps.KnowledgeConnectors.Statuses() { + if rt := s.knowledgeConnectors(); rt != nil { + for _, st := range rt.Statuses() { byName[st.Name] = st } } @@ -327,7 +328,7 @@ func (s *Server) handleKnowledgeConnectorsStatus(w http.ResponseWriter, r *http. sort.SliceStable(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name }) jsonResponse(w, map[string]interface{}{ "connectors": rows, - "runtime": s.deps.KnowledgeConnectors != nil, + "runtime": s.knowledgeConnectors() != nil, "publish": s.knowledgePublishView(), }) } @@ -407,7 +408,7 @@ func (s *Server) handleKnowledgeConnectorsSync(w http.ResponseWriter, r *http.Re jsonError(w, "unknown connector", http.StatusNotFound) return } - rt := s.deps.KnowledgeConnectors + rt := s.knowledgeConnectors() if rt == nil { jsonError(w, "connector syncer is not running (restart the hive after enabling knowledge connectors)", http.StatusServiceUnavailable) return diff --git a/src/pkg/dashboard/api_knowledge_connectors.go b/src/pkg/dashboard/api_knowledge_connectors.go index 35de58ad19..481f69f8b1 100644 --- a/src/pkg/dashboard/api_knowledge_connectors.go +++ b/src/pkg/dashboard/api_knowledge_connectors.go @@ -5,6 +5,7 @@ import ( "net/http" "time" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/knowledge/connector" ) @@ -91,9 +92,33 @@ func (s *Server) knowledgeConnectors() KnowledgeConnectorRuntime { if s == nil || s.deps == nil { return nil } + s.knowledgeConnectorsRTMu.RLock() + defer s.knowledgeConnectorsRTMu.RUnlock() return s.deps.KnowledgeConnectors } +func (s *Server) setKnowledgeConnectors(rt KnowledgeConnectorRuntime) { + s.knowledgeConnectorsRTMu.Lock() + defer s.knowledgeConnectorsRTMu.Unlock() + s.deps.KnowledgeConnectors = rt +} + +// reloadKnowledgeConnectors hot-loads entries into the connector runtime +// after a dashboard save (#18091), so a connector added, edited, enabled, +// disabled or removed takes effect without restarting the hive. A failure +// is logged and leaves the previous runtime in place. +func (s *Server) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) { + if s.deps.ReloadKnowledgeConnectors == nil { + return + } + rt, err := s.deps.ReloadKnowledgeConnectors(entries) + if err != nil { + s.logger.Warn("knowledge connectors not reloaded; restart the hive to apply", "error", err) + return + } + s.setKnowledgeConnectors(rt) +} + // knowledgeConnectorStatus finds one connector's status by name. func knowledgeConnectorStatus(rt KnowledgeConnectorRuntime, name string) (connector.Status, bool) { for _, st := range rt.Statuses() { diff --git a/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go b/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go new file mode 100644 index 0000000000..9401dd37e6 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go @@ -0,0 +1,121 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "strconv" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// hotloadDocConn stands in for the document connector so a hot-loaded sync +// never touches the network. +type hotloadDocConn struct{ blockingConn } + +func (*hotloadDocConn) Type() string { return connector.TypeDocument } + +// withHotloadRuntime wires ReloadKnowledgeConnectors the way boot does: build +// the syncer on first use, reconfigure it afterwards. +func withHotloadRuntime(t *testing.T, s *Server) { + t.Helper() + reg := connector.NewRegistry() + if err := reg.Register(connector.TypeDocument, func(connector.ConnectorConfig, connector.Deps) (connector.Connector, error) { + return &hotloadDocConn{}, nil + }); err != nil { + t.Fatal(err) + } + vault := t.TempDir() + var syncer *connector.Syncer + s.deps.ReloadKnowledgeConnectors = func(entries []config.KnowledgeConnector) (KnowledgeConnectorRuntime, error) { + cfgs := make([]connector.ConnectorConfig, 0, len(entries)) + for _, e := range entries { + rc, err := knowledgeConnectorRuntimeConfig(e) + if err != nil { + return nil, err + } + cfgs = append(cfgs, rc) + } + if syncer != nil { + return syncer, syncer.Reconfigure(cfgs) + } + if len(cfgs) == 0 { + return nil, nil + } + built, err := connector.NewSyncer(cfgs, connector.SyncerOptions{ + Registry: reg, + VaultDir: func(string) (string, error) { return vault, nil }, + }) + if err != nil { + return nil, err + } + syncer = built + return syncer, nil + } +} + +// A connector added from the dashboard to a hive booted with none is loaded +// without a restart: it is listed, Sync answers 202, and disabling or +// removing it takes effect immediately (#18091). +func TestKnowledgeConnectorsHotLoad(t *testing.T) { + s := covApiServer(t) + withHotloadRuntime(t, s) + + if rec := doPostRaw(s, "/api/knowledge/connectors/docs/sync", ""); rec.Code != http.StatusNotFound { + t.Fatalf("sync before any connector = %d, want 404", rec.Code) + } + + put := func(enabled bool) { + t.Helper() + body := `{"connectors":[{"name":"docs","type":"document","enabled":` + strconv.FormatBool(enabled) + + `,"layer":"project","scope":{"url":"https://example.org/guide.html"}}]}` + if rec := doPutRaw(s, "/api/config/knowledge/connectors", body); rec.Code != http.StatusOK { + t.Fatalf("PUT = %d %q", rec.Code, rec.Body.String()) + } + } + put(true) + + got := decodeConnectorList(t, s) + if len(got) != 1 || got[0].Name != "docs" || !got[0].Enabled { + t.Fatalf("connectors after add = %+v, want docs enabled", got) + } + rec := doGetNoRole(s, "/api/config/knowledge/connectors/status") + var status struct { + Runtime bool `json:"runtime"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil || !status.Runtime { + t.Fatalf("status runtime = %v (%v): %s", status.Runtime, err, rec.Body.String()) + } + + if rec := doPostRaw(s, "/api/knowledge/connectors/docs/sync", ""); rec.Code != http.StatusAccepted { + t.Fatalf("sync after hot-load = %d %q, want 202", rec.Code, rec.Body.String()) + } + deadline := time.After(10 * time.Second) + for { + if st, ok := knowledgeConnectorStatus(s.knowledgeConnectors(), "docs"); ok && !st.LastSync.IsZero() && !st.Running { + break + } + select { + case <-deadline: + t.Fatal("hot-loaded connector never finished its sync") + case <-time.After(5 * time.Millisecond): + } + } + + put(false) + if got := decodeConnectorList(t, s); len(got) != 1 || got[0].Enabled { + t.Fatalf("connectors after disable = %+v, want docs disabled", got) + } + + if rec := doPutRaw(s, "/api/config/knowledge/connectors", `{"connectors":[]}`); rec.Code != http.StatusOK { + t.Fatalf("PUT empty = %d %q", rec.Code, rec.Body.String()) + } + if got := decodeConnectorList(t, s); len(got) != 0 { + t.Fatalf("connectors after remove = %+v, want none", got) + } + if rec := doPostRaw(s, "/api/knowledge/connectors/docs/sync", ""); rec.Code != http.StatusNotFound { + t.Fatalf("sync after remove = %d, want 404", rec.Code) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 8ed04dbb1a..ce7a5f7edd 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -105,8 +105,16 @@ type Dependencies struct { // (bare test deps) means the toggle can only persist the flag, so it // answers restart_required. KnowledgePrimer KnowledgePrimerControl - // KnowledgeConnectors is the running connector syncer; nil when none is wired. + // KnowledgeConnectors is the running connector syncer; nil when none is + // wired. Read it through Server.knowledgeConnectors: a dashboard edit + // can replace it at runtime (#18091). KnowledgeConnectors KnowledgeConnectorRuntime + // ReloadKnowledgeConnectors applies a saved knowledge.connectors list to + // the running hive without a restart (#18091): it builds the syncer when + // none is running, else reconfigures it in place. It returns the runtime + // to serve, nil when none is needed. Nil (bare test deps) keeps the + // pre-#18091 behaviour where a new connector needs a restart. + ReloadKnowledgeConnectors func(entries []config.KnowledgeConnector) (KnowledgeConnectorRuntime, error) // KnowledgePublish is the running knowledge.publish mirror; nil when // publishing is not configured or failed to start. KnowledgePublish KnowledgePublishRuntime diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 1c61c1dc97..cf3252946a 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -134,6 +134,10 @@ type Server struct { matrix *matrixSource matrixNudge stageTrackerNudger + // knowledgeConnectorsRTMu guards deps.KnowledgeConnectors, which a + // dashboard connector edit can swap at runtime (#18091). + knowledgeConnectorsRTMu sync.RWMutex + port int authToken string sweepMu sync.RWMutex diff --git a/src/pkg/knowledge/connector/syncer.go b/src/pkg/knowledge/connector/syncer.go index efe5efc6f7..42fe277cfb 100644 --- a/src/pkg/knowledge/connector/syncer.go +++ b/src/pkg/knowledge/connector/syncer.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "path/filepath" + "reflect" "sync" "time" ) @@ -67,19 +68,29 @@ type entry struct { cfg ConnectorConfig conn Connector buildErr error - syncMu sync.Mutex + // syncMu is shared by every entry built for the same name, so a + // reconfigured connector never syncs concurrently with its predecessor. + syncMu *sync.Mutex + // stop cancels this entry's Run loop; nil when no loop is running. + stop context.CancelFunc } // Syncer runs every enabled connector on its interval and writes the pages -// it emits into the vault for the connector's layer. +// it emits into the vault for the connector's layer. Reconfigure adds, +// replaces and removes connectors while it runs (#18091). type Syncer struct { - opts SyncerOptions - logger *slog.Logger + opts SyncerOptions + logger *slog.Logger + + // mu guards order, entries, status and runCtx. + mu sync.Mutex order []string entries map[string]*entry - - mu sync.Mutex - status map[string]Status + status map[string]Status + // runCtx is Run's context once Run has started; loops for connectors + // added later by Reconfigure derive from it. + runCtx context.Context + loops sync.WaitGroup } // NewSyncer builds connectors for cfgs. A connector that fails to build is @@ -104,26 +115,14 @@ func NewSyncer(cfgs []ConnectorConfig, opts SyncerOptions) (*Syncer, error) { entries: map[string]*entry{}, status: map[string]Status{}, } + if err := checkDuplicateNames(cfgs); err != nil { + return nil, err + } persisted := s.loadState() for _, cfg := range cfgs { - if _, dup := s.entries[cfg.Name]; dup { - return nil, fmt.Errorf("connector: duplicate name %q", cfg.Name) - } - deps := opts.Deps - if deps.StateDir != "" { - deps.StateDir = filepath.Join(deps.StateDir, cfg.Name) - } - e := &entry{cfg: cfg} - e.conn, e.buildErr = opts.Registry.New(cfg, deps) st := persisted[cfg.Name] - st.Name, st.Type, st.Layer = cfg.Name, cfg.Type, cfg.Layer - st.Enabled = cfg.Enabled - st.Interval = cfg.EffectiveInterval().String() st.Running = false - if e.buildErr != nil { - st.LastError = e.buildErr.Error() - s.logger.Warn("knowledge connector disabled: invalid config", "name", cfg.Name, "type", cfg.Type, "error", e.buildErr) - } + e := s.newEntry(cfg, &sync.Mutex{}, &st) s.entries[cfg.Name] = e s.order = append(s.order, cfg.Name) s.status[cfg.Name] = st @@ -131,6 +130,92 @@ func NewSyncer(cfgs []ConnectorConfig, opts SyncerOptions) (*Syncer, error) { return s, nil } +func checkDuplicateNames(cfgs []ConnectorConfig) error { + seen := make(map[string]bool, len(cfgs)) + for _, cfg := range cfgs { + if seen[cfg.Name] { + return fmt.Errorf("connector: duplicate name %q", cfg.Name) + } + seen[cfg.Name] = true + } + return nil +} + +// newEntry builds the connector for cfg and refreshes st's config fields. +// A build error is recorded in st.LastError. +func (s *Syncer) newEntry(cfg ConnectorConfig, syncMu *sync.Mutex, st *Status) *entry { + deps := s.opts.Deps + if deps.StateDir != "" { + deps.StateDir = filepath.Join(deps.StateDir, cfg.Name) + } + e := &entry{cfg: cfg, syncMu: syncMu} + e.conn, e.buildErr = s.opts.Registry.New(cfg, deps) + st.Name, st.Type, st.Layer = cfg.Name, cfg.Type, cfg.Layer + st.Enabled = cfg.Enabled + st.Interval = cfg.EffectiveInterval().String() + if e.buildErr != nil { + st.LastError = e.buildErr.Error() + s.logger.Warn("knowledge connector disabled: invalid config", "name", cfg.Name, "type", cfg.Type, "error", e.buildErr) + } + return e +} + +// Reconfigure applies cfgs to a built (and possibly running) syncer without a +// restart (#18091): new names are added, entries whose config changed are +// rebuilt, and names no longer listed are removed. Unchanged entries keep +// their connector and Run loop. Status (cursor, counts) carries over for a +// changed entry, as it does across a hive restart. Duplicate names are an +// error and leave the syncer unchanged. +func (s *Syncer) Reconfigure(cfgs []ConnectorConfig) error { + if err := checkDuplicateNames(cfgs); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + keep := make(map[string]bool, len(cfgs)) + order := make([]string, 0, len(cfgs)) + for _, cfg := range cfgs { + keep[cfg.Name] = true + order = append(order, cfg.Name) + old, ok := s.entries[cfg.Name] + if ok && reflect.DeepEqual(old.cfg, cfg) { + continue + } + syncMu := &sync.Mutex{} + st := s.status[cfg.Name] + if ok { + s.stopLoopLocked(old) + syncMu = old.syncMu + if old.buildErr != nil && st.LastError == old.buildErr.Error() { + st.LastError = "" + } + } + e := s.newEntry(cfg, syncMu, &st) + s.entries[cfg.Name] = e + s.status[cfg.Name] = st + s.startLoopLocked(cfg.Name, e) + s.logger.Info("knowledge connector loaded", "name", cfg.Name, "type", cfg.Type, "enabled", cfg.Enabled, "replaced", ok) + } + for _, name := range s.order { + if keep[name] { + continue + } + s.stopLoopLocked(s.entries[name]) + delete(s.entries, name) + delete(s.status, name) + s.logger.Info("knowledge connector removed", "name", name) + } + s.order = order + return nil +} + +func (s *Syncer) lookup(name string) (*entry, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e, ok := s.entries[name] + return e, ok +} + // Statuses returns a snapshot of every connector's status, in config order. func (s *Syncer) Statuses() []Status { s.mu.Lock() @@ -155,14 +240,18 @@ func (s *Syncer) update(name string, fn func(*Status)) Status { defer s.mu.Unlock() st := s.status[name] fn(&st) - s.status[name] = st + // A sync that finishes after Reconfigure removed its connector must + // not resurrect the removed status. + if _, ok := s.entries[name]; ok { + s.status[name] = st + } return st } // Connector returns the built connector for name; ok is false for an // unknown name or one whose config failed to build. func (s *Syncer) Connector(name string) (Connector, bool) { - e, ok := s.entries[name] + e, ok := s.lookup(name) if !ok || e.buildErr != nil || e.conn == nil { return nil, false } @@ -172,10 +261,16 @@ func (s *Syncer) Connector(name string) (Connector, bool) { // SyncNow runs one sync of the named connector immediately, whether or not // it is enabled. It returns ErrSyncInProgress if a sync is already running. func (s *Syncer) SyncNow(ctx context.Context, name string) (Status, error) { - e, ok := s.entries[name] + e, ok := s.lookup(name) if !ok { return Status{}, fmt.Errorf("%w: %q", ErrUnknownConnector, name) } + return s.syncEntry(ctx, name, e) +} + +// syncEntry runs one sync of e. A Run loop passes its own entry so a loop +// cancelled by Reconfigure never syncs the entry that replaced it. +func (s *Syncer) syncEntry(ctx context.Context, name string, e *entry) (Status, error) { if e.buildErr != nil { st, _ := s.Status(name) return st, e.buildErr @@ -293,26 +388,51 @@ func (s *Syncer) runSync(ctx context.Context, e *entry, cur Cursor) (int, Cursor } // Run syncs every enabled, valid connector immediately and then on its -// interval until ctx is cancelled. It blocks until all loops have exited. +// interval until ctx is cancelled; connectors added or changed later by +// Reconfigure start their own loop. It blocks until ctx is cancelled and +// every loop has exited. func (s *Syncer) Run(ctx context.Context) { - var wg sync.WaitGroup + s.mu.Lock() + s.runCtx = ctx for _, name := range s.order { - e := s.entries[name] - if !e.cfg.Enabled || e.buildErr != nil { - continue - } - wg.Add(1) - go func(name string, interval time.Duration) { - defer wg.Done() - for { - _, _ = s.SyncNow(ctx, name) - if sleepCtx(ctx, interval) != nil { - return - } + s.startLoopLocked(name, s.entries[name]) + } + s.mu.Unlock() + <-ctx.Done() + s.loops.Wait() +} + +// startLoopLocked starts e's interval loop when Run is active and e is +// enabled and valid. s.mu must be held. +func (s *Syncer) startLoopLocked(name string, e *entry) { + if s.runCtx == nil || s.runCtx.Err() != nil || !e.cfg.Enabled || e.buildErr != nil { + return + } + ctx, cancel := context.WithCancel(s.runCtx) + e.stop = cancel + interval := e.cfg.EffectiveInterval() + s.loops.Add(1) + go func() { + defer s.loops.Done() + // Wait out a sync still running for a replaced entry of the same + // name so the new config's first sync is not skipped as in progress. + e.syncMu.Lock() + e.syncMu.Unlock() //nolint:staticcheck // SA2001: intentional barrier + for ctx.Err() == nil { + _, _ = s.syncEntry(ctx, name, e) + if sleepCtx(ctx, interval) != nil { + return } - }(name, e.cfg.EffectiveInterval()) + } + }() +} + +// stopLoopLocked cancels e's interval loop, if any. s.mu must be held. +func (s *Syncer) stopLoopLocked(e *entry) { + if e != nil && e.stop != nil { + e.stop() + e.stop = nil } - wg.Wait() } type persistedStatus struct { diff --git a/src/pkg/knowledge/connector/syncer_reconfigure_test.go b/src/pkg/knowledge/connector/syncer_reconfigure_test.go new file mode 100644 index 0000000000..434c67a0bb --- /dev/null +++ b/src/pkg/knowledge/connector/syncer_reconfigure_test.go @@ -0,0 +1,157 @@ +package connector + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +func statusNames(s *Syncer) []string { + var out []string + for _, st := range s.Statuses() { + out = append(out, st.Name) + } + return out +} + +// Reconfigure adds, replaces and removes connectors in place (#18091). +func TestSyncerReconfigure(t *testing.T) { + a, b := &fakeConn{typ: "fake"}, &fakeConn{typ: "fake"} + s, _ := newTestSyncer(t, []ConnectorConfig{fakeCfg("a")}, map[string]Connector{"a": a, "b": b}, nil) + if _, err := s.SyncNow(context.Background(), "a"); err != nil { + t.Fatal(err) + } + + changed := fakeCfg("a") + changed.Interval = 2 * time.Hour + if err := s.Reconfigure([]ConnectorConfig{fakeCfg("b"), changed}); err != nil { + t.Fatal(err) + } + if got := strings.Join(statusNames(s), ","); got != "b,a" { + t.Fatalf("statuses = %s, want b,a (config order)", got) + } + st, _ := s.Status("a") + if st.Interval != (2 * time.Hour).String() { + t.Fatalf("changed interval not applied: %+v", st) + } + if st.LastSync.IsZero() { + t.Fatal("a changed connector must keep its sync status, as across a restart") + } + if _, err := s.SyncNow(context.Background(), "b"); err != nil { + t.Fatalf("added connector not syncable: %v", err) + } + + if err := s.Reconfigure([]ConnectorConfig{fakeCfg("b")}); err != nil { + t.Fatal(err) + } + if got := strings.Join(statusNames(s), ","); got != "b" { + t.Fatalf("statuses = %s, want b", got) + } + if _, err := s.SyncNow(context.Background(), "a"); !errors.Is(err, ErrUnknownConnector) { + t.Fatalf("removed connector SyncNow err = %v, want ErrUnknownConnector", err) + } + if _, ok := s.Connector("a"); ok { + t.Fatal("removed connector still resolvable") + } + + if err := s.Reconfigure([]ConnectorConfig{fakeCfg("b"), fakeCfg("b")}); err == nil || !strings.Contains(err.Error(), "duplicate name") { + t.Fatalf("duplicate err = %v", err) + } + if got := strings.Join(statusNames(s), ","); got != "b" { + t.Fatalf("a rejected Reconfigure changed the syncer: %s", got) + } +} + +// A connector whose config failed to build recovers once Reconfigure gives it +// a buildable config, and its stale build error is cleared. +func TestSyncerReconfigureClearsBuildError(t *testing.T) { + s, _ := newTestSyncer(t, []ConnectorConfig{fakeCfg("x")}, map[string]Connector{}, nil) + if st, _ := s.Status("x"); st.LastError == "" { + t.Fatal("expected a build error for x") + } + // Swap the factory's answer by re-registering under a new type. + if err := s.opts.Registry.Register("fake2", func(ConnectorConfig, Deps) (Connector, error) { + return &fakeConn{typ: "fake2"}, nil + }); err != nil { + t.Fatal(err) + } + fixed := fakeCfg("x") + fixed.Type = "fake2" + if err := s.Reconfigure([]ConnectorConfig{fixed}); err != nil { + t.Fatal(err) + } + if st, _ := s.Status("x"); st.LastError != "" || st.Type != "fake2" { + t.Fatalf("status after fix = %+v", st) + } + if _, ok := s.Connector("x"); !ok { + t.Fatal("fixed connector not built") + } +} + +// While Run is active, Reconfigure starts loops for added connectors and +// stops loops for removed or disabled ones. +func TestSyncerReconfigureWhileRunning(t *testing.T) { + a, b := &fakeConn{typ: "fake"}, &fakeConn{typ: "fake"} + s, _ := newTestSyncer(t, nil, map[string]Connector{"a": a, "b": b}, nil) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + s.Run(ctx) + close(done) + }() + waitCalls := func(c *fakeConn, n int32) { + t.Helper() + deadline := time.After(10 * time.Second) + for c.calls.Load() < n { + select { + case <-deadline: + t.Fatalf("connector synced %d times, want >= %d", c.calls.Load(), n) + case <-time.After(time.Millisecond): + } + } + } + + fast := fakeCfg("a") + fast.Interval = 5 * time.Millisecond + if err := s.Reconfigure([]ConnectorConfig{fast}); err != nil { + t.Fatal(err) + } + waitCalls(a, 2) + + off := fast + off.Enabled = false + if err := s.Reconfigure([]ConnectorConfig{off}); err != nil { + t.Fatal(err) + } + // a was the only loop, so s.loops drains exactly when disabling it has + // stopped its loop (after at most one in-flight sync). + stopped := make(chan struct{}) + go func() { + s.loops.Wait() + close(stopped) + }() + select { + case <-stopped: + case <-time.After(10 * time.Second): + t.Fatal("disabled connector's loop did not stop") + } + before := a.calls.Load() + + if err := s.Reconfigure([]ConnectorConfig{off, fakeCfg("b")}); err != nil { + t.Fatal(err) + } + waitCalls(b, 1) + if after := a.calls.Load(); after != before { + t.Fatalf("disabled connector kept syncing: %d -> %d", before, after) + } + + cancel() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Run did not return after cancel") + } +} From c62a010a3cec8bcd1db84805dd693228f302e800 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 20:30:34 +0000 Subject: [PATCH 172/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.246.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.246.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-18061-translator-proxy-uid.md | 1 - .../fixed-18121-contributor-lane-incomplete-files.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-18061-translator-proxy-uid.md delete mode 100644 changelog.d/fixed-18121-contributor-lane-incomplete-files.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0747a9d687..2f98ee2427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.246.1) + +### Fixed + +- Stop refusing inference requests from the hive's own UID (`dev`, 1001 on v5 pods) when private plain-HTTP forwarding is off, so agents without a per-agent UID reach their inference route again; the proxy UID is still refused when `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` (#18061). +- The contributor-lane automerge sweep now treats a PR with more changed files than GitHub's files API returns (3000) as a standing `files-evidence` skip instead of re-warning every sweep. + ## 2026-11-13 (v5.246.0) ### Added diff --git a/changelog.d/fixed-18061-translator-proxy-uid.md b/changelog.d/fixed-18061-translator-proxy-uid.md deleted file mode 100644 index a4d62970b6..0000000000 --- a/changelog.d/fixed-18061-translator-proxy-uid.md +++ /dev/null @@ -1 +0,0 @@ -- Stop refusing inference requests from the hive's own UID (`dev`, 1001 on v5 pods) when private plain-HTTP forwarding is off, so agents without a per-agent UID reach their inference route again; the proxy UID is still refused when `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` (#18061). diff --git a/changelog.d/fixed-18121-contributor-lane-incomplete-files.md b/changelog.d/fixed-18121-contributor-lane-incomplete-files.md deleted file mode 100644 index a823e5198f..0000000000 --- a/changelog.d/fixed-18121-contributor-lane-incomplete-files.md +++ /dev/null @@ -1 +0,0 @@ -- The contributor-lane automerge sweep now treats a PR with more changed files than GitHub's files API returns (3000) as a standing `files-evidence` skip instead of re-warning every sweep. From 2732d6e93b7e2551c93e1f0c3b0319c5ba74ded3 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 15:57:39 -0500 Subject: [PATCH 173/223] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20show=20out-?= =?UTF-8?q?of-quota=20agents=20red=20instead=20of=20"authenticated"=20(#18?= =?UTF-8?q?116)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 dashboard: show out-of-quota agents red instead of "authenticated" An agent refused by its provider for quota (e.g. Copilot "You have exceeded your monthly quota") stayed green and "authenticated" on the dashboard: crux.Process.QuotaExhausted was never copied to the agent row, and the tile/card state logic had no quota case. - /api/status agent rows carry quotaExhausted (omitempty) from proc.QuotaExhausted; openapi.json documents it. - New agentOutOfQuota(a) helper (quotaExhausted or backendAuthStatus 'quota') drives the top-bar tile, collapsed tile and agent card: red, labelled "out of quota", with a card note that never suggests re-login. Logged-out/expired outrank quota; disabled, paused and stopped agents keep their look. Fixes #18111 Hive-Run: hivecommons/hive#18111 Hive-Plan: issue-18111 "How to build it" steps 1-6 Hive-Spec: issue-18111#wanted-behavior Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🐛 dashboard: one out-of-quota predicate for card and tiles; tokenize note CSS Review on #18116: agentCollapsedAuthInfo painted an out-of-quota agent red on the top-bar and collapsed tiles even when it was held off by the current mode (offByCadence / unscheduledInMode), while the card kept its off look. Add agentOutOfQuotaShown(a), which applies the card's exact disabled / paused / held-off / starting / stopped rules, and have the card and both tiles call it. Login problems still rank first at every call site. TestStyleRatchet failed because .agent-quota-note added a raw padding and a raw font-size; use --sp-3/--sp-5 and --fs-xs instead. Tests: node cases for held-off, starting, on-demand+paused and the shared predicate; the render contract pins the tile call; the agentHeldByMode isOff site count goes from 4 to 5 for the new helper. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 📖 docs: re-anchor api-reference server.go citations after #18111 shift The quotaExhausted field adds 5 lines to server.go ahead of the route table, so 16 api-reference citations pointed at stale lines. Rewritten with check-api-reference-citations.sh --fix on top of v5 @ 2cc699516. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18111-agent-out-of-quota-red.md | 1 + dashboard/openapi.json | 4 + src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +-- src/docs/api-reference/snapshots-and-style.md | 2 +- .../agent_card_mode_unscheduled_7474_test.go | 4 +- src/pkg/dashboard/agent_quota_state_test.go | 186 ++++++++++++++++++ src/pkg/dashboard/server.go | 5 + .../dashboard/static-src/css/agent-cards.css | 9 + .../dashboard/static-src/css/agent-fields.css | 1 + .../static-src/css/sections-collapse.css | 8 +- .../dashboard/static-src/js/agent-cards.js | 40 +++- src/pkg/dashboard/static-src/js/sections.js | 16 +- src/pkg/dashboard/static/index.html | 74 ++++++- src/pkg/dashboard/status_builder.go | 3 + 16 files changed, 345 insertions(+), 38 deletions(-) create mode 100644 changelog.d/fixed-18111-agent-out-of-quota-red.md create mode 100644 src/pkg/dashboard/agent_quota_state_test.go diff --git a/changelog.d/fixed-18111-agent-out-of-quota-red.md b/changelog.d/fixed-18111-agent-out-of-quota-red.md new file mode 100644 index 0000000000..3905c6437a --- /dev/null +++ b/changelog.d/fixed-18111-agent-out-of-quota-red.md @@ -0,0 +1 @@ +- Dashboard: an agent whose provider refused it for quota (for example Copilot's "You have exceeded your monthly quota") is now drawn red and labelled `out of quota` on the top-bar tile, the collapsed Agents tile and its agent card, instead of green "authenticated"; `/api/status` agent rows carry `quotaExhausted: true` from the existing per-agent quota flag. Login problems still show first, and disabled, paused and stopped agents keep their own look ([#18111](https://github.com/hivecommons/hive/issues/18111)). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 9d1e34573a..14efdd4274 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -1631,6 +1631,10 @@ "backendAuthLastError": { "type": "string", "description": "The offending pane line that produced backendAuthStatus." + }, + "quotaExhausted": { + "type": "boolean", + "description": "True when the agent's pane shows a provider quota refusal (e.g. Copilot \"You have exceeded your monthly quota\"), from crux.Process.QuotaExhausted; covers every backend (#18111). Omitted when false." } } } diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index dfd32421ab..3213dc2ac3 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1716` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1717` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1718` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1721` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1722` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1723` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index e301461e4e..9009a479cf 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1723` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1728` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index a1cb297651..32337f311a 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1696` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1697` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1698` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1704` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1709` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1710` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1711` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1712` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1713` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1714` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1715` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1701` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1702` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1703` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1709` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1714` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1715` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1716` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1717` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1718` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1719` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1720` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 8dcc35479a..b657c1836b 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1786` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1791` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/agent_card_mode_unscheduled_7474_test.go b/src/pkg/dashboard/agent_card_mode_unscheduled_7474_test.go index 5503de3d19..5acca84f9d 100644 --- a/src/pkg/dashboard/agent_card_mode_unscheduled_7474_test.go +++ b/src/pkg/dashboard/agent_card_mode_unscheduled_7474_test.go @@ -161,8 +161,8 @@ func TestAgentCardModeUnscheduledStructure(t *testing.T) { if strings.Contains(html, "const isOff = a.offByCadence === true && !isPaused && !isOnDemand;") { t.Error("an isOff site still reads offByCadence alone; it must go through agentHeldByMode(a) or the unscheduled auditor renders green-idle there") } - if n := strings.Count(html, "const isOff = agentHeldByMode(a) && !isPaused && !isOnDemand;"); n != 4 { - t.Errorf("expected the card grid, the ops-center nav, the detail panel and the Now-line state helper (4 sites) to derive isOff from agentHeldByMode, found %d", n) + if n := strings.Count(html, "const isOff = agentHeldByMode(a) && !isPaused && !isOnDemand;"); n != 5 { + t.Errorf("expected the card grid, the ops-center nav, the detail panel, the Now-line state helper and agentOutOfQuotaShown (#18111) (5 sites) to derive isOff from agentHeldByMode, found %d", n) } if strings.Contains(html, "title=\"${esc(OFF_HEALTHY_TITLE)}\"") { t.Error("an off-healthy dot still uses the paused-only OFF_HEALTHY_TITLE; it must use agentOffTitle(a), which says WHICH gap holds the agent") diff --git a/src/pkg/dashboard/agent_quota_state_test.go b/src/pkg/dashboard/agent_quota_state_test.go new file mode 100644 index 0000000000..1ecb5392ee --- /dev/null +++ b/src/pkg/dashboard/agent_quota_state_test.go @@ -0,0 +1,186 @@ +package dashboard + +import ( + "encoding/json" + "os/exec" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +// #18111: the pane-poll quota flag (crux.Process.QuotaExhausted) must reach +// the /api/status agent row for every backend, not only gateway agents whose +// classifier sets BackendAuthStatus "quota". +func TestBuildFrontendStatus_QuotaExhaustedReachesAgentRow(t *testing.T) { + cfg := &config.Config{ + AgentRuntimeConfig: config.AgentRuntimeConfig{ + Agents: map[string]config.AgentConfig{ + "scanner": {Backend: "copilot", Model: "gpt-5", Enabled: true}, + "quality": {Backend: "copilot", Model: "gpt-5", Enabled: true}, + }, + }, + } + gov := governor.New(cfg.Governor, cfg.Agents, nil) + statuses := map[string]*crux.Process{ + "scanner": { + Name: "scanner", + Config: cfg.Agents["scanner"], + State: crux.StateRunning, + OutputBuffer: cruxtest.NewRingBuffer(10), + QuotaExhausted: true, + }, + "quality": { + Name: "quality", + Config: cfg.Agents["quality"], + State: crux.StateRunning, + OutputBuffer: cruxtest.NewRingBuffer(10), + }, + } + + payload := BuildFrontendStatus(gov.GetState(), nil, statuses, cfg, nil, gov, nil, nil, nil, nil) + rows := map[string]FrontendAgent{} + for _, a := range payload.Agents { + rows[a.Name] = a + } + if !rows["scanner"].QuotaExhausted { + t.Fatal("scanner row QuotaExhausted = false, want true from crux.Process.QuotaExhausted") + } + if rows["scanner"].BackendAuthStatus != "" { + t.Fatalf("scanner BackendAuthStatus = %q, want empty: the quota flag must not rewrite the classifier verdict", rows["scanner"].BackendAuthStatus) + } + if rows["quality"].QuotaExhausted { + t.Fatal("quality row QuotaExhausted = true, want false") + } + + raw, err := json.Marshal(rows["scanner"]) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), `"quotaExhausted":true`) { + t.Fatalf("scanner row JSON missing quotaExhausted:true: %s", raw) + } + raw, err = json.Marshal(rows["quality"]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "quotaExhausted") { + t.Fatalf("quality row JSON must omit quotaExhausted when false: %s", raw) + } +} + +// #18111: an out-of-quota agent is red and says "out of quota" on the top-bar +// tile, the collapsed tile and the agent card; login problems outrank it and +// disabled/paused/held-off/starting/stopped agents keep their own look on +// every surface (one shared predicate, agentOutOfQuotaShown). +func TestAgentOutOfQuotaTileStates(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node unavailable: agent out-of-quota tile states were not executed") + } + html := indexHTML(t) + script := `const assert = require('node:assert/strict'); +function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"'); } +` + jsFunc(t, html, "agentIsDisabled") + ` +` + jsFunc(t, html, "_agentNeedsLogin") + ` +` + jsFunc(t, html, "agentOutOfQuota") + ` +` + jsFunc(t, html, "agentHeldByMode") + ` +` + jsFunc(t, html, "agentSandboxRun") + ` +` + jsFunc(t, html, "agentSessionDown") + ` +` + jsFunc(t, html, "agentOutOfQuotaShown") + ` +` + jsFunc(t, html, "agentCollapsedAuthInfo") + ` +` + jsFunc(t, html, "agentCollapsedStateLabel") + ` +` + jsFunc(t, html, "agentOutOfQuotaNoteHtml") + ` +const live = { name: 'scanner', state: 'running', enabled: true }; +let auth = agentCollapsedAuthInfo({ ...live, quotaExhausted: true }, false); +assert.deepEqual(auth, { label: 'out of quota', status: 'error', needsLogin: false, quota: true }); +assert.equal(agentCollapsedStateLabel(false, false, auth), 'out of quota'); + +auth = agentCollapsedAuthInfo({ ...live, backendAuthStatus: 'quota' }, false); +assert.equal(auth.label, 'out of quota'); +assert.equal(auth.status, 'error'); + +auth = agentCollapsedAuthInfo({ ...live, quotaExhausted: true, needsLogin: true }, false); +assert.equal(auth.label, 'logged out'); +assert.equal(auth.quota, undefined); + +auth = agentCollapsedAuthInfo({ ...live, enabled: false, quotaExhausted: true }, false); +assert.equal(auth.status, 'ok', 'disabled agent must not be painted red for quota'); +assert.equal(auth.quota, undefined); + +auth = agentCollapsedAuthInfo({ ...live, paused: true, quotaExhausted: true }, false); +assert.equal(auth.status, 'ok', 'paused agent must not be painted red for quota'); + +auth = agentCollapsedAuthInfo({ ...live, state: 'stopped', quotaExhausted: true }, true); +assert.equal(auth.quota, undefined, 'stopped agent keeps its stopped look'); + +// Held off by the current mode: the card keeps its off look, so the tiles +// must not paint it red either (review on #18116). +auth = agentCollapsedAuthInfo({ ...live, offByCadence: true, quotaExhausted: true }, false); +assert.equal(auth.quota, undefined, 'offByCadence agent must not be red on the tile'); +assert.equal(auth.status, 'ok'); +auth = agentCollapsedAuthInfo({ ...live, unscheduledInMode: true, quotaExhausted: true }, false); +assert.equal(auth.quota, undefined, 'unscheduledInMode agent must not be red on the tile'); + +// Shared predicate, matching the card's state rules. +const q = { ...live, quotaExhausted: true }; +assert.equal(agentOutOfQuotaShown(q), true); +assert.equal(agentOutOfQuotaShown({ ...q, offByCadence: true }), false, 'held off'); +assert.equal(agentOutOfQuotaShown({ ...q, state: 'starting', starting: true }), false, 'starting'); +assert.equal(agentOutOfQuotaShown({ ...q, state: 'stopped' }), false, 'stopped'); +assert.equal(agentOutOfQuotaShown({ ...q, paused: true }), false, 'paused'); +assert.equal(agentOutOfQuotaShown({ ...q, enabled: false }), false, 'disabled'); +// On-demand: a stale paused flag does not hide quota, same as the card. +assert.equal(agentOutOfQuotaShown({ ...q, onDemand: true, paused: true }), true, 'on-demand ignores paused'); +assert.equal(agentOutOfQuotaShown({ ...live }), false, 'no quota signal'); + +auth = agentCollapsedAuthInfo({ ...live }, false); +assert.deepEqual(auth, { label: 'authenticated', status: 'ok', needsLogin: false }); + +assert.equal(agentOutOfQuota(null), false); +assert.equal(agentOutOfQuota({ backendAuthStatus: 'token-expired' }), false); + +let note = agentOutOfQuotaNoteHtml({ backendAuthLastError: 'You have exceeded your monthly quota' }); +assert.match(note, /Out of provider quota: this agent cannot run until the quota resets or it is moved to another provider\./); +assert.match(note, /You have exceeded your monthly quota/); +assert.doesNotMatch(note, /log ?in/i); +note = agentOutOfQuotaNoteHtml({}); +assert.doesNotMatch(note, /agent-quota-note-detail/); +` + out, err := exec.Command(node, "-e", script).CombinedOutput() + if err != nil { + t.Fatalf("agent out-of-quota tile states failed: %v\n%s", err, strings.TrimSpace(string(out))) + } +} + +func TestAgentOutOfQuotaRenderContracts(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + // card: quota state after disabled/login, never over paused/off/stopped + "const outOfQuota = !needsLogin && agentOutOfQuotaShown(a);", + "needsLogin ? 'needs-login' : outOfQuota ? 'out-of-quota' :", + "isStopped ? 'stopped' : outOfQuota ? 'out-of-quota' :", + "${outOfQuota ? agentOutOfQuotaNoteHtml(a) : ''}", + // tiles share the card's predicate + "if (!isDown && agentOutOfQuotaShown(a)) return", + "auth.quota ? 'out-of-quota' : ''", + "`login: ${auth.quota ? 'authenticated' : auth.label}`", + "...(auth.quota ? ['provider: out of quota'] : [])", + // CSS uses existing red tokens + ".agent-card.out-of-quota { border-left-color: var(--red); }", + ".dot.out-of-quota { background: var(--status-error); }", + ".agent-collapsed-tile.out-of-quota .agent-tile-dot", + ".agent-navbar-tile.out-of-quota .agent-tile-dot", + ".agent-navbar-tile.stopped, .agent-navbar-tile.needs-login, .agent-navbar-tile.out-of-quota { border-color", + } { + if !strings.Contains(html, want) { + t.Errorf("out-of-quota render contract missing %q", want) + } + } + if strings.Count(html, "auth.quota ? 'out-of-quota' : ''") != 2 { + t.Error("both the collapsed tile and the top-bar tile must add the out-of-quota class") + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index cf3252946a..079050c4cf 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -1021,6 +1021,11 @@ type FrontendAgent struct { // token, false = a human must log in. Absent for backends with no // verified refresh semantics. BackendAuthRefreshable *bool `json:"backendAuthRefreshable,omitempty"` + // QuotaExhausted mirrors crux.Process.QuotaExhausted (#18111): the pane + // poll saw a provider quota refusal (e.g. Copilot "exceeded your monthly + // quota"). Unlike BackendAuthStatus "quota" it covers every backend, so + // the dashboard can paint the agent "out of quota" instead of green. + QuotaExhausted bool `json:"quotaExhausted,omitempty"` // InferenceLastError is the full text of the most recent inference error // hive's proxy returned to this agent (#11761), shown while Claude Code's // own retry line is still clipped to one pane row. Empty when unknown, diff --git a/src/pkg/dashboard/static-src/css/agent-cards.css b/src/pkg/dashboard/static-src/css/agent-cards.css index 51171d5fe6..f1be4ba1e5 100644 --- a/src/pkg/dashboard/static-src/css/agent-cards.css +++ b/src/pkg/dashboard/static-src/css/agent-cards.css @@ -36,6 +36,15 @@ .agent-card.idle { border-left-color: var(--line); } .agent-card.stopped { border-left-color: var(--red); } .agent-card.needs-login { border-left-color: var(--red); } + .agent-card.out-of-quota { border-left-color: var(--red); } + /* #18111: out of provider quota — red, and no login affordance. */ + .agent-quota-note { + margin: var(--sp-4) var(--sp-0) var(--sp-5); padding: var(--sp-3) var(--sp-5); + border: 1px solid color-mix(in srgb, var(--red) 55%, transparent); + border-radius: var(--radius); background: color-mix(in srgb, var(--red) 10%, var(--panel)); + color: var(--text); font-size: var(--fs-xs); line-height: 1.45; overflow-wrap: anywhere; + } + .agent-quota-note-detail { color: var(--muted); } .agent-card.disabled-agent { border-left-color: var(--status-neutral); opacity: 0.68; } .agent-card.disabled-agent .agent-state { color: color-mix(in srgb, var(--muted) 70%, var(--bg)); } .terminal-copy-hint { diff --git a/src/pkg/dashboard/static-src/css/agent-fields.css b/src/pkg/dashboard/static-src/css/agent-fields.css index d3d1976f8e..56eda913ba 100644 --- a/src/pkg/dashboard/static-src/css/agent-fields.css +++ b/src/pkg/dashboard/static-src/css/agent-fields.css @@ -21,6 +21,7 @@ the current mode, so it will not be kicked (mirrors needs-login, green). */ .dot.off-healthy { background: transparent; border: 2px solid var(--status-ok); box-sizing: border-box; } .dot.stopped { background: var(--status-error); } + .dot.out-of-quota { background: var(--status-error); } .dot.starting { background: var(--status-info); animation: oc-pulse 1.5s ease-in-out infinite; } .dot.needs-login { background: transparent; border: 2px solid var(--status-attention); box-sizing: border-box; } .agent-field { display: grid; grid-template-columns: minmax(5.5rem, max-content) minmax(0, 1fr); gap: var(--sp-4); align-items: baseline; font-size: 0.8rem; padding: var(--sp-1) var(--sp-0); } diff --git a/src/pkg/dashboard/static-src/css/sections-collapse.css b/src/pkg/dashboard/static-src/css/sections-collapse.css index cb73116104..ff2ba623d5 100644 --- a/src/pkg/dashboard/static-src/css/sections-collapse.css +++ b/src/pkg/dashboard/static-src/css/sections-collapse.css @@ -87,7 +87,7 @@ .agent-collapsed-tile:hover, .agent-collapsed-tile:focus-visible, .repo-collapsed-tile:hover, .repo-collapsed-tile:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } .agent-collapsed-tile.disabled, .repo-collapsed-tile.paused, .repo-collapsed-tile.idle { opacity: 0.58; } .agent-collapsed-tile.stopped, .repo-collapsed-tile.error { border-color: color-mix(in srgb, var(--red) 35%, var(--line-subtle)); } - .agent-collapsed-tile.needs-login { border-color: color-mix(in srgb, var(--red) 55%, var(--line-subtle)); } + .agent-collapsed-tile.needs-login, .agent-collapsed-tile.out-of-quota { border-color: color-mix(in srgb, var(--red) 55%, var(--line-subtle)); } .agent-collapsed-tile.active, .repo-collapsed-tile.healthy { border-color: color-mix(in srgb, var(--green) 70%, var(--line-subtle)); } .repo-collapsed-tile.blocked { border-color: color-mix(in srgb, var(--yellow) 65%, var(--line-subtle)); } .agent-collapsed-tile.up-next, .repo-collapsed-tile.working { box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--accent) 55%, transparent); } @@ -102,7 +102,7 @@ .agent-tile-next { color: var(--muted); grid-column: 1 / -1; } .agent-tile-dot { width: 7px; height: 7px; border-radius: 50%; background: var(--green); box-shadow: 0 0 0 1px color-mix(in srgb, var(--green) 40%, transparent); } .agent-collapsed-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } - .agent-collapsed-tile.stopped .agent-tile-dot, .agent-collapsed-tile.needs-login .agent-tile-dot, .repo-collapsed-tile.error .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } + .agent-collapsed-tile.stopped .agent-tile-dot, .agent-collapsed-tile.needs-login .agent-tile-dot, .agent-collapsed-tile.out-of-quota .agent-tile-dot, .repo-collapsed-tile.error .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } .repo-collapsed-tile.blocked .agent-tile-dot { background: var(--yellow); box-shadow: 0 0 0 1px color-mix(in srgb, var(--yellow) 40%, transparent); } .agent-collapsed-tile.disabled .agent-tile-dot, .repo-collapsed-tile.paused .agent-tile-dot, .repo-collapsed-tile.idle .agent-tile-dot { background: var(--muted); box-shadow: none; } .agent-navbar-upnext { @@ -119,14 +119,14 @@ .agent-navbar-tile.entering { opacity: 0; transform: translateY(calc(-1 * var(--sp-3))); } .agent-navbar-tile.exiting { opacity: 0; transform: translateY(var(--sp-3)); pointer-events: none; } .agent-navbar-tile.active { border-color: color-mix(in srgb, var(--green) 70%, var(--line-subtle)); } - .agent-navbar-tile.stopped, .agent-navbar-tile.needs-login { border-color: color-mix(in srgb, var(--red) 45%, var(--line-subtle)); } + .agent-navbar-tile.stopped, .agent-navbar-tile.needs-login, .agent-navbar-tile.out-of-quota { border-color: color-mix(in srgb, var(--red) 45%, var(--line-subtle)); } .agent-navbar-tile .agent-tile-dot { grid-column: 2; grid-row: 1; justify-self: end; } .agent-navbar-tile .agent-tile-name { grid-column: 1; grid-row: 1; min-width: max-content; overflow: visible; text-overflow: clip; } .agent-navbar-tile .agent-tile-next { grid-column: 1; grid-row: 2; color: var(--muted); } .agent-navbar-tile .agent-up-next { grid-column: 2; grid-row: 2; color: var(--amber); } .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } - .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } + .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot, .agent-navbar-tile.out-of-quota .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } .agent-navbar-summary { flex: 0 0 auto; display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); cursor: pointer; font: inherit; font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } .agent-navbar-summary:hover, .agent-navbar-summary:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } .agent-navbar-summary-short { display: none; } diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index 5253df7de5..f72670ee33 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -296,6 +296,13 @@ sentence — that one reports whether a login URL was actually seen, so it earns the space and always wins over the persisted dismissal. */ const TERMINAL_KEYBOARD_COPY_HINT = 'Selecting text in the terminal: hold ⇧ (Shift) while dragging — tmux mouse mode owns an ordinary drag. Then ⌘C (macOS) or Ctrl+Shift+C copies, and ⌘V or Ctrl+Shift+V pastes.'; + /* One-line card note for an out-of-quota agent (#18111). It never offers a + login: quota is a provider limit, and re-login does not lift it. */ + function agentOutOfQuotaNoteHtml(a) { + const detail = String((a && a.backendAuthLastError) || '').trim(); + const detailHtml = detail ? ` <span class="agent-quota-note-detail">${esc(detail)}</span>` : ''; + return `<div class="agent-quota-note" role="note">Out of provider quota: this agent cannot run until the quota resets or it is moved to another provider.${detailHtml}</div>`; + } function terminalCopyHintHtml(needsLogin, sessionUnavailable, agentName, loginURL, backendAuthStatus, backendAuthLastError, statusEvidence, inferenceLastError) { if (sessionUnavailable) return ''; /* #11761: inferenceLastError is the FULL error hive's inference proxy @@ -652,6 +659,30 @@ const down = (isDown === undefined) ? (a.state !== 'running') : isDown; return down && a.authKnown === true && a.authAvailable !== true; } + /* agentOutOfQuota (#18111): the provider refused this agent for quota. + quotaExhausted is the pane-poll flag that covers every backend (e.g. + Copilot's "exceeded your monthly quota"); backendAuthStatus 'quota' is + the inference-gateway classifier's verdict. The top-bar tile, collapsed + tile and agent card all ask this one helper so they cannot disagree. + Re-login does not help, so callers keep login problems ahead of it. */ + function agentOutOfQuota(a) { + return !!a && (a.quotaExhausted === true || a.backendAuthStatus === 'quota'); + } + /* agentOutOfQuotaShown: whether the out-of-quota red is painted at all. + Disabled, paused, held-off-by-mode, starting and stopped agents keep + their own look. The card and both tiles call this one predicate, with + the same paused/off/starting/stopped rules the card uses, so a held-off + or starting agent cannot be red on a tile and normal on its card. + Login problems are not checked here; every caller ranks them first. */ + function agentOutOfQuotaShown(a) { + if (!agentOutOfQuota(a) || agentIsDisabled(a)) return false; + const isOnDemand = a.onDemand === true; + const isPaused = a.paused === true && !isOnDemand; + const isOff = agentHeldByMode(a) && !isPaused && !isOnDemand; + const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; + const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; + return !isPaused && !isOff && !isStarting && !isStopped; + } /* Agent blockers (#5594). THREE orthogonal flags decide whether an agent will ever do work: is its session up, has an operator paused it, and is @@ -886,6 +917,10 @@ const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; const needsLogin = _agentNeedsLogin(a, isStopped); + // #18111: out of provider quota paints a live agent red. Disabled, + // paused, held-off, starting and stopped agents keep their own look, + // and a login problem outranks quota (re-login is the first fix). + const outOfQuota = !needsLogin && agentOutOfQuotaShown(a); const STALE_MS = 1200000; // 20 minutes const isStale = a.summaryUpdated && !isPaused && a.busy === 'working' && (Date.now() - new Date(a.summaryUpdated).getTime()) > STALE_MS; let statusCls = ''; @@ -893,10 +928,10 @@ else if (a.structuredStatus === 'NEEDS_CONTEXT') statusCls = 'status-needs-context'; else if (isStale) statusCls = 'status-stale'; const isOnDemandIdle = isOnDemand && a.busy !== 'working'; - const cls = isDisabled ? 'disabled-agent' : needsLogin ? 'needs-login' : isStarting ? 'starting' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : isStopped ? 'stopped' : `${a.busy} ${statusCls}`.trim(); + const cls = isDisabled ? 'disabled-agent' : needsLogin ? 'needs-login' : outOfQuota ? 'out-of-quota' : isStarting ? 'starting' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : isStopped ? 'stopped' : `${a.busy} ${statusCls}`.trim(); // Down + unauthenticated CLI = attention-token needs-login (same formula as the // card's Login button above), not plain down-red (#2465). - const dotCls = isPoweredOff ? 'off' : (isStopped && needsLogin) ? 'needs-login' : isStarting ? 'starting' : isStopped ? 'stopped' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy === 'working' ? 'running' : 'idle'; + const dotCls = isPoweredOff ? 'off' : (isStopped && needsLogin) ? 'needs-login' : isStarting ? 'starting' : isStopped ? 'stopped' : outOfQuota ? 'out-of-quota' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy === 'working' ? 'running' : 'idle'; // Healthy-but-governor-paused → hollow green dot (see detail-header renderer). const dotState = isOff ? 'off-healthy' : dotCls; const canToggle = a.beadRole !== 'supervisor'; @@ -972,6 +1007,7 @@ ${agentSpanHandle} <span class="working-indicator"></span> <div class="agent-name">${agentOrderHandle}<span class="dot ${dotState}" data-status="${statusAttr(dotState)}"${isOff ? ` title="${esc(agentOffTitle(a))}"` : ''}></span>${a.emoji ? a.emoji + ' ' : ''}${esc(a.displayName || a.name)}${needsLogin ? ' <span title="CLI needs login">\uD83D\uDD11</span>' : ''}${a.sandboxed ? ' ' + statusBadgeHTML('ready', 'sandboxed', 'Sandboxed runner active', 'done') : ''}${disabledBadge}${continuousBadge}${replicaBadge} ${agentPowerSwitchHtml(a)} ${isDisabled ? disabledEnableBtn : ''}${toggleBtn}${neverScheduledChipHtml(a, configTarget)} <button class="compact-toggle" data-action="toggleAgentCompact" data-arg0="${esc(a.name)}" data-stop="1" title="Toggle compact/expanded view for this agent card">${isAgentCompact(a.name) ? '▷ expand' : '▽ compact'}</button> ${sessionActions} <button class="config-gear" data-action="openConfigDialog" data-config-type="agent" data-agent="${esc(configTarget)}" title="Configure ${esc(a.replicaBase ? (a.replicaBase + ' (base for ' + a.name + ')') : (a.displayName || a.name))}">⚙️</button></div> + ${outOfQuota ? agentOutOfQuotaNoteHtml(a) : ''} ${terminalCopyHintHtml(needsLogin, agentSessionUnavailable(a), a.name, a.loginURL, a.backendAuthStatus, a.backendAuthLastError, a.statusEvidence, a.inferenceLastError)} <div class="agent-state ${isPoweredOff ? 'disabled' : isOnDemand ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy}">${agentStateText(a)}${pauseInfoIcon(a)}${(() => { if (a.structuredStatus === 'BLOCKED') return statusBadgeHTML('blocked', 'blocked', a.statusEvidence, 'blocked'); diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index 85b6a94e2b..671dbc836d 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -46,6 +46,7 @@ needs_attention: { label: 'needs attention', status: 'attention' }, needs_context: { label: 'needs context', status: 'warn' }, needs_login: { label: 'needs login', status: 'attention' }, + out_of_quota: { label: 'out of quota', status: 'error' }, on_demand: { label: 'on demand', status: 'info' }, active: { label: 'active', status: 'info' }, initializing: { label: 'initializing', status: 'info' }, @@ -969,6 +970,11 @@ const text = String((a && (a.backendAuthStatus || a.backendAuthLastError || a.statusEvidence)) || '').toLowerCase(); if (text.includes('expired') || text.includes('token_expired')) return { label: 'expired', status: 'error', needsLogin: true }; if (_agentNeedsLogin(a || {}, isDown)) return { label: 'logged out', status: 'error', needsLogin: true }; + // #18111: out of provider quota is red, but never "logged out" — re-login + // does not help. agentOutOfQuotaShown applies the card's exclusions + // (disabled, paused, held off by mode, starting, stopped); a down tile + // already reads "stopped". + if (!isDown && agentOutOfQuotaShown(a)) return { label: 'out of quota', status: 'error', needsLogin: false, quota: true }; return { label: 'authenticated', status: 'ok', needsLogin: false }; } function agentCollapsedTimestampDetail(ms, fallback) { @@ -988,7 +994,8 @@ return [ name, `power: ${power}`, - `login: ${auth.label}`, + `login: ${auth.quota ? 'authenticated' : auth.label}`, + ...(auth.quota ? ['provider: out of quota'] : []), `next kick: ${agentCollapsedTimestampDetail(nextMs, a && (a.nextKickAt || a.nextKick))}`, `last kick: ${agentCollapsedTimestampDetail(Number.isFinite(lastMs) ? lastMs : null, a && a.lastKick)}`, ].join('\n'); @@ -997,6 +1004,7 @@ if (disabled) return 'disabled'; if (auth && auth.needsLogin) return auth.label; if (isDown) return 'stopped'; + if (auth && auth.quota) return auth.label; return ''; } function agentTileLabelHtml(label) { @@ -1030,7 +1038,7 @@ const state = agentCollapsedStateLabel(disabled, isDown, auth); const sortKey = tileState.sortKey || agentCollapsedSortKey(a, 0, nowMs); const label = tileState.label || null; - const classes = ['agent-collapsed-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', disabled ? 'disabled' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); + const classes = ['agent-collapsed-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', disabled ? 'disabled' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', auth.quota ? 'out-of-quota' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); const title = agentCollapsedTileTitle(a || {}, name, power, auth, nextMs); const labelHtml = agentTileLabelHtml(label); const stateHtml = state ? `<span class="agent-tile-state"> · ${esc(state)}</span>` : '<span class="agent-tile-state"></span>'; @@ -1095,8 +1103,8 @@ const nextMs = Number.isFinite(tileState.nextMs) ? tileState.nextMs : agentCollapsedNextKickMs(a, nowMs); const countdown = tileState.countdown || (active ? agentTileRunningLabel(a, nowMs) : agentTileCadenceLabel(a, nowMs)); const label = tileState.label || null; - const classes = ['agent-navbar-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); - const title = `${name} + const classes = ['agent-navbar-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', auth.quota ? 'out-of-quota' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); + const title = `${name}${auth.quota ? '\nprovider: out of quota' : ''} next kick: ${agentCollapsedTimestampDetail(nextMs, a && (a.nextKickAt || a.nextKick))}`; const labelHtml = agentTileLabelHtml(label); return `<button type="button" class="${classes}" data-action="openAgentsUpNextPanel" data-agent-key="${esc(key)}" data-agent-next-kick-ms="${Number.isFinite(nextMs) ? String(nextMs) : ''}" data-agent-running="${active ? '1' : ''}" data-agent-running-since="${esc((a && (a.busySince || a.runningSince || a.startedAt)) || '')}" data-agent-can-up-next="${tileState.canBeNext ? '1' : ''}" data-agent-tile-label="${esc(label || '')}" data-agent-active="${active ? '1' : ''}" data-stop="1" title="${esc(title)}" aria-label="${esc(title)}"><span class="agent-tile-name">${esc(name)}</span><span class="agent-tile-dot" data-status="${auth.status}" aria-hidden="true"></span><span class="agent-tile-next"><span class="agent-tile-countdown">${esc(countdown)}</span></span>${labelHtml}</button>`; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 15d51756ac..1646e1c9a0 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -986,6 +986,15 @@ .agent-card.idle { border-left-color: var(--line); } .agent-card.stopped { border-left-color: var(--red); } .agent-card.needs-login { border-left-color: var(--red); } + .agent-card.out-of-quota { border-left-color: var(--red); } + /* #18111: out of provider quota — red, and no login affordance. */ + .agent-quota-note { + margin: var(--sp-4) var(--sp-0) var(--sp-5); padding: var(--sp-3) var(--sp-5); + border: 1px solid color-mix(in srgb, var(--red) 55%, transparent); + border-radius: var(--radius); background: color-mix(in srgb, var(--red) 10%, var(--panel)); + color: var(--text); font-size: var(--fs-xs); line-height: 1.45; overflow-wrap: anywhere; + } + .agent-quota-note-detail { color: var(--muted); } .agent-card.disabled-agent { border-left-color: var(--status-neutral); opacity: 0.68; } .agent-card.disabled-agent .agent-state { color: color-mix(in srgb, var(--muted) 70%, var(--bg)); } .terminal-copy-hint { @@ -2682,6 +2691,7 @@ the current mode, so it will not be kicked (mirrors needs-login, green). */ .dot.off-healthy { background: transparent; border: 2px solid var(--status-ok); box-sizing: border-box; } .dot.stopped { background: var(--status-error); } + .dot.out-of-quota { background: var(--status-error); } .dot.starting { background: var(--status-info); animation: oc-pulse 1.5s ease-in-out infinite; } .dot.needs-login { background: transparent; border: 2px solid var(--status-attention); box-sizing: border-box; } .agent-field { display: grid; grid-template-columns: minmax(5.5rem, max-content) minmax(0, 1fr); gap: var(--sp-4); align-items: baseline; font-size: 0.8rem; padding: var(--sp-1) var(--sp-0); } @@ -6119,7 +6129,7 @@ .agent-collapsed-tile:hover, .agent-collapsed-tile:focus-visible, .repo-collapsed-tile:hover, .repo-collapsed-tile:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } .agent-collapsed-tile.disabled, .repo-collapsed-tile.paused, .repo-collapsed-tile.idle { opacity: 0.58; } .agent-collapsed-tile.stopped, .repo-collapsed-tile.error { border-color: color-mix(in srgb, var(--red) 35%, var(--line-subtle)); } - .agent-collapsed-tile.needs-login { border-color: color-mix(in srgb, var(--red) 55%, var(--line-subtle)); } + .agent-collapsed-tile.needs-login, .agent-collapsed-tile.out-of-quota { border-color: color-mix(in srgb, var(--red) 55%, var(--line-subtle)); } .agent-collapsed-tile.active, .repo-collapsed-tile.healthy { border-color: color-mix(in srgb, var(--green) 70%, var(--line-subtle)); } .repo-collapsed-tile.blocked { border-color: color-mix(in srgb, var(--yellow) 65%, var(--line-subtle)); } .agent-collapsed-tile.up-next, .repo-collapsed-tile.working { box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--accent) 55%, transparent); } @@ -6134,7 +6144,7 @@ .agent-tile-next { color: var(--muted); grid-column: 1 / -1; } .agent-tile-dot { width: 7px; height: 7px; border-radius: 50%; background: var(--green); box-shadow: 0 0 0 1px color-mix(in srgb, var(--green) 40%, transparent); } .agent-collapsed-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } - .agent-collapsed-tile.stopped .agent-tile-dot, .agent-collapsed-tile.needs-login .agent-tile-dot, .repo-collapsed-tile.error .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } + .agent-collapsed-tile.stopped .agent-tile-dot, .agent-collapsed-tile.needs-login .agent-tile-dot, .agent-collapsed-tile.out-of-quota .agent-tile-dot, .repo-collapsed-tile.error .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } .repo-collapsed-tile.blocked .agent-tile-dot { background: var(--yellow); box-shadow: 0 0 0 1px color-mix(in srgb, var(--yellow) 40%, transparent); } .agent-collapsed-tile.disabled .agent-tile-dot, .repo-collapsed-tile.paused .agent-tile-dot, .repo-collapsed-tile.idle .agent-tile-dot { background: var(--muted); box-shadow: none; } .agent-navbar-upnext { @@ -6151,14 +6161,14 @@ .agent-navbar-tile.entering { opacity: 0; transform: translateY(calc(-1 * var(--sp-3))); } .agent-navbar-tile.exiting { opacity: 0; transform: translateY(var(--sp-3)); pointer-events: none; } .agent-navbar-tile.active { border-color: color-mix(in srgb, var(--green) 70%, var(--line-subtle)); } - .agent-navbar-tile.stopped, .agent-navbar-tile.needs-login { border-color: color-mix(in srgb, var(--red) 45%, var(--line-subtle)); } + .agent-navbar-tile.stopped, .agent-navbar-tile.needs-login, .agent-navbar-tile.out-of-quota { border-color: color-mix(in srgb, var(--red) 45%, var(--line-subtle)); } .agent-navbar-tile .agent-tile-dot { grid-column: 2; grid-row: 1; justify-self: end; } .agent-navbar-tile .agent-tile-name { grid-column: 1; grid-row: 1; min-width: max-content; overflow: visible; text-overflow: clip; } .agent-navbar-tile .agent-tile-next { grid-column: 1; grid-row: 2; color: var(--muted); } .agent-navbar-tile .agent-up-next { grid-column: 2; grid-row: 2; color: var(--amber); } .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } - .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } + .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot, .agent-navbar-tile.out-of-quota .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } .agent-navbar-summary { flex: 0 0 auto; display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); cursor: pointer; font: inherit; font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } .agent-navbar-summary:hover, .agent-navbar-summary:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } .agent-navbar-summary-short { display: none; } @@ -10511,6 +10521,7 @@ <h3 class="faq-group-title">Where to get help</h3> needs_attention: { label: 'needs attention', status: 'attention' }, needs_context: { label: 'needs context', status: 'warn' }, needs_login: { label: 'needs login', status: 'attention' }, + out_of_quota: { label: 'out of quota', status: 'error' }, on_demand: { label: 'on demand', status: 'info' }, active: { label: 'active', status: 'info' }, initializing: { label: 'initializing', status: 'info' }, @@ -11434,6 +11445,11 @@ <h3 class="faq-group-title">Where to get help</h3> const text = String((a && (a.backendAuthStatus || a.backendAuthLastError || a.statusEvidence)) || '').toLowerCase(); if (text.includes('expired') || text.includes('token_expired')) return { label: 'expired', status: 'error', needsLogin: true }; if (_agentNeedsLogin(a || {}, isDown)) return { label: 'logged out', status: 'error', needsLogin: true }; + // #18111: out of provider quota is red, but never "logged out" — re-login + // does not help. agentOutOfQuotaShown applies the card's exclusions + // (disabled, paused, held off by mode, starting, stopped); a down tile + // already reads "stopped". + if (!isDown && agentOutOfQuotaShown(a)) return { label: 'out of quota', status: 'error', needsLogin: false, quota: true }; return { label: 'authenticated', status: 'ok', needsLogin: false }; } function agentCollapsedTimestampDetail(ms, fallback) { @@ -11453,7 +11469,8 @@ <h3 class="faq-group-title">Where to get help</h3> return [ name, `power: ${power}`, - `login: ${auth.label}`, + `login: ${auth.quota ? 'authenticated' : auth.label}`, + ...(auth.quota ? ['provider: out of quota'] : []), `next kick: ${agentCollapsedTimestampDetail(nextMs, a && (a.nextKickAt || a.nextKick))}`, `last kick: ${agentCollapsedTimestampDetail(Number.isFinite(lastMs) ? lastMs : null, a && a.lastKick)}`, ].join('\n'); @@ -11462,6 +11479,7 @@ <h3 class="faq-group-title">Where to get help</h3> if (disabled) return 'disabled'; if (auth && auth.needsLogin) return auth.label; if (isDown) return 'stopped'; + if (auth && auth.quota) return auth.label; return ''; } function agentTileLabelHtml(label) { @@ -11495,7 +11513,7 @@ <h3 class="faq-group-title">Where to get help</h3> const state = agentCollapsedStateLabel(disabled, isDown, auth); const sortKey = tileState.sortKey || agentCollapsedSortKey(a, 0, nowMs); const label = tileState.label || null; - const classes = ['agent-collapsed-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', disabled ? 'disabled' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); + const classes = ['agent-collapsed-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', disabled ? 'disabled' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', auth.quota ? 'out-of-quota' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); const title = agentCollapsedTileTitle(a || {}, name, power, auth, nextMs); const labelHtml = agentTileLabelHtml(label); const stateHtml = state ? `<span class="agent-tile-state"> · ${esc(state)}</span>` : '<span class="agent-tile-state"></span>'; @@ -11560,8 +11578,8 @@ <h3 class="faq-group-title">Where to get help</h3> const nextMs = Number.isFinite(tileState.nextMs) ? tileState.nextMs : agentCollapsedNextKickMs(a, nowMs); const countdown = tileState.countdown || (active ? agentTileRunningLabel(a, nowMs) : agentTileCadenceLabel(a, nowMs)); const label = tileState.label || null; - const classes = ['agent-navbar-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); - const title = `${name} + const classes = ['agent-navbar-tile', active ? 'active' : '', label === 'running' ? 'running-now' : '', isDown && !disabled ? 'stopped' : '', auth.needsLogin ? 'needs-login' : '', auth.quota ? 'out-of-quota' : '', label === 'next' ? 'up-next' : ''].filter(Boolean).join(' '); + const title = `${name}${auth.quota ? '\nprovider: out of quota' : ''} next kick: ${agentCollapsedTimestampDetail(nextMs, a && (a.nextKickAt || a.nextKick))}`; const labelHtml = agentTileLabelHtml(label); return `<button type="button" class="${classes}" data-action="openAgentsUpNextPanel" data-agent-key="${esc(key)}" data-agent-next-kick-ms="${Number.isFinite(nextMs) ? String(nextMs) : ''}" data-agent-running="${active ? '1' : ''}" data-agent-running-since="${esc((a && (a.busySince || a.runningSince || a.startedAt)) || '')}" data-agent-can-up-next="${tileState.canBeNext ? '1' : ''}" data-agent-tile-label="${esc(label || '')}" data-agent-active="${active ? '1' : ''}" data-stop="1" title="${esc(title)}" aria-label="${esc(title)}"><span class="agent-tile-name">${esc(name)}</span><span class="agent-tile-dot" data-status="${auth.status}" aria-hidden="true"></span><span class="agent-tile-next"><span class="agent-tile-countdown">${esc(countdown)}</span></span>${labelHtml}</button>`; @@ -30902,6 +30920,13 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> sentence — that one reports whether a login URL was actually seen, so it earns the space and always wins over the persisted dismissal. */ const TERMINAL_KEYBOARD_COPY_HINT = 'Selecting text in the terminal: hold ⇧ (Shift) while dragging — tmux mouse mode owns an ordinary drag. Then ⌘C (macOS) or Ctrl+Shift+C copies, and ⌘V or Ctrl+Shift+V pastes.'; + /* One-line card note for an out-of-quota agent (#18111). It never offers a + login: quota is a provider limit, and re-login does not lift it. */ + function agentOutOfQuotaNoteHtml(a) { + const detail = String((a && a.backendAuthLastError) || '').trim(); + const detailHtml = detail ? ` <span class="agent-quota-note-detail">${esc(detail)}</span>` : ''; + return `<div class="agent-quota-note" role="note">Out of provider quota: this agent cannot run until the quota resets or it is moved to another provider.${detailHtml}</div>`; + } function terminalCopyHintHtml(needsLogin, sessionUnavailable, agentName, loginURL, backendAuthStatus, backendAuthLastError, statusEvidence, inferenceLastError) { if (sessionUnavailable) return ''; /* #11761: inferenceLastError is the FULL error hive's inference proxy @@ -31258,6 +31283,30 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const down = (isDown === undefined) ? (a.state !== 'running') : isDown; return down && a.authKnown === true && a.authAvailable !== true; } + /* agentOutOfQuota (#18111): the provider refused this agent for quota. + quotaExhausted is the pane-poll flag that covers every backend (e.g. + Copilot's "exceeded your monthly quota"); backendAuthStatus 'quota' is + the inference-gateway classifier's verdict. The top-bar tile, collapsed + tile and agent card all ask this one helper so they cannot disagree. + Re-login does not help, so callers keep login problems ahead of it. */ + function agentOutOfQuota(a) { + return !!a && (a.quotaExhausted === true || a.backendAuthStatus === 'quota'); + } + /* agentOutOfQuotaShown: whether the out-of-quota red is painted at all. + Disabled, paused, held-off-by-mode, starting and stopped agents keep + their own look. The card and both tiles call this one predicate, with + the same paused/off/starting/stopped rules the card uses, so a held-off + or starting agent cannot be red on a tile and normal on its card. + Login problems are not checked here; every caller ranks them first. */ + function agentOutOfQuotaShown(a) { + if (!agentOutOfQuota(a) || agentIsDisabled(a)) return false; + const isOnDemand = a.onDemand === true; + const isPaused = a.paused === true && !isOnDemand; + const isOff = agentHeldByMode(a) && !isPaused && !isOnDemand; + const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; + const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; + return !isPaused && !isOff && !isStarting && !isStopped; + } /* Agent blockers (#5594). THREE orthogonal flags decide whether an agent will ever do work: is its session up, has an operator paused it, and is @@ -31492,6 +31541,10 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const isStarting = !!a.starting && a.state !== 'running' && !isPaused && !isOff; const isStopped = agentSessionDown(a) && !isPaused && !isOff && !isOnDemand && !isStarting; const needsLogin = _agentNeedsLogin(a, isStopped); + // #18111: out of provider quota paints a live agent red. Disabled, + // paused, held-off, starting and stopped agents keep their own look, + // and a login problem outranks quota (re-login is the first fix). + const outOfQuota = !needsLogin && agentOutOfQuotaShown(a); const STALE_MS = 1200000; // 20 minutes const isStale = a.summaryUpdated && !isPaused && a.busy === 'working' && (Date.now() - new Date(a.summaryUpdated).getTime()) > STALE_MS; let statusCls = ''; @@ -31499,10 +31552,10 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> else if (a.structuredStatus === 'NEEDS_CONTEXT') statusCls = 'status-needs-context'; else if (isStale) statusCls = 'status-stale'; const isOnDemandIdle = isOnDemand && a.busy !== 'working'; - const cls = isDisabled ? 'disabled-agent' : needsLogin ? 'needs-login' : isStarting ? 'starting' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : isStopped ? 'stopped' : `${a.busy} ${statusCls}`.trim(); + const cls = isDisabled ? 'disabled-agent' : needsLogin ? 'needs-login' : outOfQuota ? 'out-of-quota' : isStarting ? 'starting' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : isStopped ? 'stopped' : `${a.busy} ${statusCls}`.trim(); // Down + unauthenticated CLI = attention-token needs-login (same formula as the // card's Login button above), not plain down-red (#2465). - const dotCls = isPoweredOff ? 'off' : (isStopped && needsLogin) ? 'needs-login' : isStarting ? 'starting' : isStopped ? 'stopped' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy === 'working' ? 'running' : 'idle'; + const dotCls = isPoweredOff ? 'off' : (isStopped && needsLogin) ? 'needs-login' : isStarting ? 'starting' : isStopped ? 'stopped' : outOfQuota ? 'out-of-quota' : isOnDemandIdle ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy === 'working' ? 'running' : 'idle'; // Healthy-but-governor-paused → hollow green dot (see detail-header renderer). const dotState = isOff ? 'off-healthy' : dotCls; const canToggle = a.beadRole !== 'supervisor'; @@ -31578,6 +31631,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> ${agentSpanHandle} <span class="working-indicator"></span> <div class="agent-name">${agentOrderHandle}<span class="dot ${dotState}" data-status="${statusAttr(dotState)}"${isOff ? ` title="${esc(agentOffTitle(a))}"` : ''}></span>${a.emoji ? a.emoji + ' ' : ''}${esc(a.displayName || a.name)}${needsLogin ? ' <span title="CLI needs login">\uD83D\uDD11</span>' : ''}${a.sandboxed ? ' ' + statusBadgeHTML('ready', 'sandboxed', 'Sandboxed runner active', 'done') : ''}${disabledBadge}${continuousBadge}${replicaBadge} ${agentPowerSwitchHtml(a)} ${isDisabled ? disabledEnableBtn : ''}${toggleBtn}${neverScheduledChipHtml(a, configTarget)} <button class="compact-toggle" data-action="toggleAgentCompact" data-arg0="${esc(a.name)}" data-stop="1" title="Toggle compact/expanded view for this agent card">${isAgentCompact(a.name) ? '▷ expand' : '▽ compact'}</button> ${sessionActions} <button class="config-gear" data-action="openConfigDialog" data-config-type="agent" data-agent="${esc(configTarget)}" title="Configure ${esc(a.replicaBase ? (a.replicaBase + ' (base for ' + a.name + ')') : (a.displayName || a.name))}">⚙️</button></div> + ${outOfQuota ? agentOutOfQuotaNoteHtml(a) : ''} ${terminalCopyHintHtml(needsLogin, agentSessionUnavailable(a), a.name, a.loginURL, a.backendAuthStatus, a.backendAuthLastError, a.statusEvidence, a.inferenceLastError)} <div class="agent-state ${isPoweredOff ? 'disabled' : isOnDemand ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy}">${agentStateText(a)}${pauseInfoIcon(a)}${(() => { if (a.structuredStatus === 'BLOCKED') return statusBadgeHTML('blocked', 'blocked', a.statusEvidence, 'blocked'); diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 6433f6cba7..15040fa8a4 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1270,6 +1270,9 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config a.BackendAuthLastError = preferFullInferenceError(a.BackendAuthLastError, inferenceLastErr) } } + // #18111: the pane-poll quota flag covers every backend (Copilot's + // "exceeded your monthly quota" never reaches BackendAuthStatus). + a.QuotaExhausted = proc.QuotaExhausted if refreshableFn := getAgentRefreshableFn(); refreshableFn != nil { if refreshable, known := refreshableFn(name); known { a.BackendAuthRefreshable = &refreshable From 2aaddf6bcca9514893bbd4194db8c223c41826de Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 21:02:51 +0000 Subject: [PATCH 174/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.247.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.247.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-18091-connector-hotload.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-18091-connector-hotload.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f98ee2427..e1e66482f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.247.0) + +### Added + +- Knowledge connectors added, edited, enabled, disabled or removed from the Knowledge Integrations dashboard now take effect without restarting the hive: a hive booted with no connectors builds and starts the syncer on the first save (Check, Sync and Connect stop answering `no knowledge connectors are configured`), and a running syncer is reconfigured in place so unchanged connectors keep syncing. The `knowledge.publish` mirror still needs a restart when its connector changes ([#18091](https://github.com/hivecommons/hive/issues/18091)). + ## 2026-11-13 (v5.246.1) ### Fixed diff --git a/changelog.d/added-18091-connector-hotload.md b/changelog.d/added-18091-connector-hotload.md deleted file mode 100644 index fa62426074..0000000000 --- a/changelog.d/added-18091-connector-hotload.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge connectors added, edited, enabled, disabled or removed from the Knowledge Integrations dashboard now take effect without restarting the hive: a hive booted with no connectors builds and starts the syncer on the first save (Check, Sync and Connect stop answering `no knowledge connectors are configured`), and a running syncer is reconfigured in place so unchanged connectors keep syncing. The `knowledge.publish` mirror still needs a restart when its connector changes ([#18091](https://github.com/hivecommons/hive/issues/18091)). From 8cdab3d3f0e0d341d172bbf6770b8a8f8cc95bf0 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 21:33:11 +0000 Subject: [PATCH 175/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.247.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.247.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-18111-agent-out-of-quota-red.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-18111-agent-out-of-quota-red.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e66482f7..5677187707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.247.1) + +### Fixed + +- Dashboard: an agent whose provider refused it for quota (for example Copilot's "You have exceeded your monthly quota") is now drawn red and labelled `out of quota` on the top-bar tile, the collapsed Agents tile and its agent card, instead of green "authenticated"; `/api/status` agent rows carry `quotaExhausted: true` from the existing per-agent quota flag. Login problems still show first, and disabled, paused and stopped agents keep their own look ([#18111](https://github.com/hivecommons/hive/issues/18111)). + ## 2026-11-13 (v5.247.0) ### Added diff --git a/changelog.d/fixed-18111-agent-out-of-quota-red.md b/changelog.d/fixed-18111-agent-out-of-quota-red.md deleted file mode 100644 index 3905c6437a..0000000000 --- a/changelog.d/fixed-18111-agent-out-of-quota-red.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: an agent whose provider refused it for quota (for example Copilot's "You have exceeded your monthly quota") is now drawn red and labelled `out of quota` on the top-bar tile, the collapsed Agents tile and its agent card, instead of green "authenticated"; `/api/status` agent rows carry `quotaExhausted: true` from the existing per-agent quota flag. Login problems still show first, and disabled, paused and stopped agents keep their own look ([#18111](https://github.com/hivecommons/hive/issues/18111)). From 9bdcb5532f6565899e8c1062b84d2b6b518facf3 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:47:34 +0000 Subject: [PATCH 176/223] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stop=20repeating?= =?UTF-8?q?=20session=20prune=20permission-denied=20warnings=20(#18130)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: stop repeating session prune permission-denied warnings Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * test: keep session contents so repeated prune passes fail again Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- .../fixed-session-prune-permission-denied.md | 1 + src/pkg/sessionprune/prune.go | 13 +++++++ src/pkg/sessionprune/prune_test.go | 39 +++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 changelog.d/fixed-session-prune-permission-denied.md diff --git a/changelog.d/fixed-session-prune-permission-denied.md b/changelog.d/fixed-session-prune-permission-denied.md new file mode 100644 index 0000000000..de23533df1 --- /dev/null +++ b/changelog.d/fixed-session-prune-permission-denied.md @@ -0,0 +1 @@ +- Session prune now warns once per path when a session directory cannot be removed due to permission denied, instead of repeating the warning every tick. diff --git a/src/pkg/sessionprune/prune.go b/src/pkg/sessionprune/prune.go index 4e99579c05..7cd4418cbd 100644 --- a/src/pkg/sessionprune/prune.go +++ b/src/pkg/sessionprune/prune.go @@ -19,9 +19,15 @@ import ( "log/slog" "os" "path/filepath" + "sync" "time" ) +// deniedWarned remembers paths whose removal already failed with a permission +// error. Prune runs every tick and a path that is not ours to delete stays +// that way, so warning each pass floods the log (hivecommons/hive#18126). +var deniedWarned sync.Map + // DefaultRetentionDays is how long a session directory is kept after its last // write. Sessions feed token accounting and post-hoc debugging, both of which // look at recent activity, so a week is ample. @@ -91,6 +97,13 @@ func Prune(dir string, maxAge time.Duration, now time.Time, logger *slog.Logger) if err := os.RemoveAll(path); err != nil { res.Failed++ + if errors.Is(err, fs.ErrPermission) { + // Warn once per path, then drop to debug. + if _, seen := deniedWarned.LoadOrStore(path, struct{}{}); seen { + logger.Debug("session prune: remove failed (permission denied, already reported)", "path", path, "err", err) + continue + } + } logger.Warn("session prune: remove failed", "path", path, "err", err) continue } diff --git a/src/pkg/sessionprune/prune_test.go b/src/pkg/sessionprune/prune_test.go index 5c8de09c3c..0da1ef1905 100644 --- a/src/pkg/sessionprune/prune_test.go +++ b/src/pkg/sessionprune/prune_test.go @@ -1,10 +1,12 @@ package sessionprune import ( + "bytes" "io" "log/slog" "os" "path/filepath" + "strings" "testing" "time" ) @@ -282,3 +284,40 @@ func TestPruneCountsFailedRemovals(t *testing.T) { t.Errorf("session directory reported as present but is gone: %v", err) } } + +// TestPruneWarnsOncePerPermissionDeniedPath ensures a path that keeps failing +// with a permission error is warned about once, not on every prune pass. +func TestPruneWarnsOncePerPermissionDeniedPath(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: permission bits do not deny removal") + } + root := t.TempDir() + old := time.Now().Add(-30 * 24 * time.Hour) + session := mkSession(t, root, "bbbbbbbb-bbbb-cccc-dddd-eeeeeeeeeeee", old, old) + // Lock the session directory too: otherwise RemoveAll deletes its contents + // before the final unlink fails, which bumps the directory mtime and makes + // later passes skip it instead of failing again. + if err := os.Chmod(session, 0o500); err != nil { + t.Fatalf("chmod session: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(session, 0o755) }) + if err := os.Chmod(root, 0o500); err != nil { + t.Fatalf("chmod root: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(root, 0o755) }) + + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn})) + for i := 0; i < 3; i++ { + res, err := Prune(root, 7*24*time.Hour, time.Now(), logger) + if err != nil { + t.Fatalf("Prune: %v", err) + } + if res.Failed != 1 { + t.Fatalf("pass %d: Failed = %d, want 1", i, res.Failed) + } + } + if n := strings.Count(buf.String(), "remove failed"); n != 1 { + t.Errorf("warnings = %d, want 1; log:\n%s", n, buf.String()) + } +} From 0e486c9d4bce1c62d691ca0799db4ab1283b1b01 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 22:18:03 +0000 Subject: [PATCH 177/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.247.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.247.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-session-prune-permission-denied.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-session-prune-permission-denied.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5677187707..2006a9c374 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.247.2) + +### Fixed + +- Session prune now warns once per path when a session directory cannot be removed due to permission denied, instead of repeating the warning every tick. + ## 2026-11-13 (v5.247.1) ### Fixed diff --git a/changelog.d/fixed-session-prune-permission-denied.md b/changelog.d/fixed-session-prune-permission-denied.md deleted file mode 100644 index de23533df1..0000000000 --- a/changelog.d/fixed-session-prune-permission-denied.md +++ /dev/null @@ -1 +0,0 @@ -- Session prune now warns once per path when a session directory cannot be removed due to permission denied, instead of repeating the warning every tick. From a2cb35ab2e4ceccc95029f3c629e098d723ba034 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:32:44 +0000 Subject: [PATCH 178/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20document=20revie?= =?UTF-8?q?w.perspectives=5Fsource=20(#18078)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [scanner] docs: document review.perspectives_source Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * [scanner] docs: cover startup/reload resolution of review.perspectives_source Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: a denied perspectives_source skips the last-known-good cache The resolver returns before consulting the cache when the source repo is not allowlisted, so a denied source falls straight to the inline perspectives. Say so instead of listing it with the cached fallbacks. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Andy Anderson <andy@clubanderson.com> --- .../added-17975-review-perspectives-source.md | 1 + src/docs/agent-configuration.md | 2 ++ src/docs/review-swarm.md | 32 +++++++++++++++++++ src/hive.yaml.example | 9 ++++++ 4 files changed, 44 insertions(+) create mode 100644 changelog.d/added-17975-review-perspectives-source.md diff --git a/changelog.d/added-17975-review-perspectives-source.md b/changelog.d/added-17975-review-perspectives-source.md new file mode 100644 index 0000000000..caffb45297 --- /dev/null +++ b/changelog.d/added-17975-review-perspectives-source.md @@ -0,0 +1 @@ +- **`review.perspectives_source` is documented.** `src/docs/review-swarm.md` now describes linking the review perspective set to a `kind: ReviewPerspectives` file in a GitHub repo (names and focus text only; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay operator-owned), the source-wins/inline-fallback rule, the last-known-good fallback order with a warning on a malformed document, the shared seed-only `github_prompt_allowlist` trust gate, and the read-only Review Gate display. `hive.yaml.example` gains a commented `perspectives_source:` block ([#17975](https://github.com/hivecommons/hive/issues/17975), [#17990](https://github.com/hivecommons/hive/issues/17990)). diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index fbff124ed5..9b909897ae 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1078,6 +1078,8 @@ This is the property operators most need to understand before enabling the featu An empty allowlist denies every repo even with `allow_github_prompt: true` — the allowlist is required, not merely advisory. +`review.perspectives_source` (the live-linked review perspective set; see [review-swarm.md](review-swarm.md#live-linked-perspectives-reviewperspectives_source)) uses the same seed-only gate: the resolver is allowed to read a repo only when `GitHubPromptAllowed` permits it, it resolves at the same resolve points (startup and every reload), and it follows the same fallback shape as `Resolve` above (last-known-good, then the inline config). + ### Fetch failures never blank an agent `Resolve` (`defsrc.go:283`) never propagates an error to the caller: a reload must proceed even when GitHub is unreachable. On any failure it falls back, in order: diff --git a/src/docs/review-swarm.md b/src/docs/review-swarm.md index b200d5424a..d1cd6a4782 100644 --- a/src/docs/review-swarm.md +++ b/src/docs/review-swarm.md @@ -31,6 +31,37 @@ review: - `perspective_prompts` overrides the focus line for a built-in, or defines a hive-specific perspective entirely (custom names: lowercase letters, digits, single dashes, ≤ 40 chars). A blank entry means the built-in text. - `combined_perspectives` reviews every enabled perspective in **one** agent session that leaves **one** review comment, with findings grouped under the perspective they belong to. Off, each perspective is its own session and its own comment, and `max_perspectives_per_pr` caps how many a PR receives. Combined mode ignores that cap: it exists to bound comments, and a combined review is always exactly one. The reviewer still emits one verdict per perspective — as a JSON array in the verdict file — so any single perspective can still hold a PR for a human. +### Live-linked perspectives (`review.perspectives_source`) + +`review.perspectives_source` links the perspective set to a file in a GitHub repo, so maintainers add, remove and re-describe perspectives through ordinary PRs instead of a hive config edit (hivecommons/hive#17975). It has the same shape as an agent's `definition_source` (`owner`, `repo`, `path`, optional `ref` that defaults to the repo's default branch; a source missing `owner`, `repo` or `path` counts as unset): + +```yaml +review: + combined_perspectives: true # stays operator-owned + perspectives_source: + owner: my-org + repo: my-repo + path: .hive/review-perspectives.yaml + ref: main # optional +``` + +```yaml +# .hive/review-perspectives.yaml +kind: ReviewPerspectives +perspectives: + - name: correctness # built-in; no focus = built-in text + - name: security + - name: my-constraints # hive-defined; focus is required + focus: "Head size a multiple of 64; fp16 only." +``` + +- **Names and focus only.** The file holds the perspective names, in dispatch order, and each one's optional focus text, validated exactly like inline `perspectives` / `perspective_prompts` (built-in names may omit `focus`; custom names need it; duplicates, an empty list, a wrong `kind` or a document over 512 KiB are rejected). `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` are cost and dispatch settings and stay operator-owned in the hive config. +- **The source wins; inline is the fallback.** While the source resolves, its set replaces inline `review.perspectives` and `review.perspective_prompts` everywhere the effective set is read (`ReviewConfig.EffectivePerspectives`). Inline values stay in the config and apply whenever nothing resolves. +- **Fallback order.** A fetch that has no GitHub client or fails, and a document that is malformed, each keep the last-known-good document for that source. With none cached, the inline set applies, and if that is empty too, the built-ins. A source whose repo is not on the allowlist is ignored outright (logged, nothing fetched or cached) and the inline set applies. A malformed document keeps the last-known-good set and records a warning rather than quietly dropping to the five built-ins. Only a document that parsed cleanly is remembered, so the cache never holds a corrupt set. A fetch is bounded to 8 seconds. +- **When it resolves.** Resolution happens at startup, before the first relay install, and again on every config reload. Each time the relay's perspective set is reinstalled, so a verdict naming a removed perspective is refused and an added one is accepted without a restart. Heads waiting on a removed perspective settle rather than stay stuck (#14905). A dashboard edit of the review config keeps the linked set. +- **Same trust gate as `definition_source`.** The source repo must be on the seed-only `variables.security.github_prompt_allowlist`; see [agent-configuration.md](agent-configuration.md#the-trust-boundary-allowlisted-repos-are-seed-only). +- **Dashboard.** Governor → Features → Review Gate → Perspectives shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line, and the resolver's status and any warning. The perspective checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next resolve would replace. `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. + ### Combined perspectives: recommended, opt-in **Turn `combined_perspectives: true` on** (Governor → Features → Review Gate, or the config above) unless you have a reason not to. It is the single largest review-swarm token saving available (hivecommons/hive#12126): @@ -181,6 +212,7 @@ review: fan_out: true max_parallel_reviews: 5 reviewer_agents: [reviewer-a, reviewer-b] # optional; otherwise agents with review role/keywords are selected + # perspectives_source: {owner: my-org, repo: my-repo, path: .hive/review-perspectives.yaml} # optional; live-link the perspective set, see "Live-linked perspectives" fixer_agent: scanner # optional; defaults to the PR lane, then scanner fixer_fallback_agents: [] # opt-in ordered fallback lanes, e.g. [fixer, scanner-2] fixer_fallback_after_undeliverable: 3 # 0 defaults to 3 only when fallbacks are configured diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 2dcfdf7cf7..9486e24393 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -884,6 +884,15 @@ governor: # combined_perspectives: true # RECOMMENDED (#12126): one session + one comment covering every # # perspective; shared review boilerplate is sent once instead of once # # per perspective. Verdicts stay per-perspective. Default false. +# perspectives_source: # optional; live-link the perspective names + focus text to a +# owner: my-org # kind: ReviewPerspectives file in GitHub (#17975). When it resolves it +# repo: my-repo # wins over perspectives/perspective_prompts, which stay the fallback. +# path: .hive/review-perspectives.yaml +# ref: main # optional; default branch otherwise +# # The repo must be on variables.security.github_prompt_allowlist (seed-only). +# # File: "kind: ReviewPerspectives" + "perspectives: [{name, focus}]". Names and +# # focus only; combined_perspectives, plan_match and max_perspectives_per_pr stay +# # here. Format: src/docs/review-swarm.md, "Live-linked perspectives". # fixer_agent: "" # optional override; defaults to the PR lane, then scanner # fixer_fallback_agents: [] # opt-in ordered repair lanes, e.g. [fixer, scanner-2] # fixer_fallback_after_undeliverable: 3 # consecutive refusals before rerouting; From 1c8c604195566ad9b2ef9bd37b13cc51ead089b8 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 17:37:44 -0500 Subject: [PATCH 179/223] =?UTF-8?q?=E2=9C=A8=20knowledge:=20refresh=20inbo?= =?UTF-8?q?x=20sources=20and=20surface=20reload=20errors=20after=20connect?= =?UTF-8?q?or=20hot-load=20(#18135)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #18099 (Syncer.Reconfigure hot-load), porting the two pieces #18100 had that #18099 left out: - The Inbox follows the live connector set. boot.reloadKnowledgeConnectors rebuilds the Inbox sources (inbox.Service.SetSources) after every dashboard save that reached the runtime, so a Plaud/WisprFlow connector added through the dashboard feeds GET /api/inbox at once, and removing the last one drops it. bootKnowledgeConnectors does the same once the boot syncer exists, so the direct Plaud read path (PreferDirect) is attached at boot instead of the fact-only fallback the Inbox had because it boots ahead of the syncer. - A save the running syncer refused is reported. The PUT still persists the config, but the response now carries runtime_error verbatim (plus runtime, with the status endpoint's meaning), and kcSave rejects with 'saved, but the running syncer did not reload: ...' so the toast says so instead of a plain success. Refs #18091 Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...nnector-hotload-inbox-and-reload-errors.md | 1 + src/cmd/hive/boot_inbox.go | 54 ++++++++---- src/cmd/hive/boot_inbox_sources_test.go | 84 +++++++++++++++++++ src/cmd/hive/boot_knowledge_connectors.go | 14 +++- .../api_config_knowledge_connectors.go | 13 ++- src/pkg/dashboard/api_knowledge_connectors.go | 11 ++- .../api_knowledge_connectors_hotload_test.go | 55 ++++++++++++ .../knowledge_connector_reload_error.test.js | 83 ++++++++++++++++++ .../dashboard/static-src/js/config-tabs.js | 3 + src/pkg/dashboard/static/index.html | 3 + src/pkg/inbox/service.go | 18 ++++ src/pkg/inbox/service_sources_test.go | 42 ++++++++++ 12 files changed, 359 insertions(+), 22 deletions(-) create mode 100644 changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md create mode 100644 src/cmd/hive/boot_inbox_sources_test.go create mode 100644 src/pkg/dashboard/knowledge_connector_reload_error.test.js create mode 100644 src/pkg/inbox/service_sources_test.go diff --git a/changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md b/changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md new file mode 100644 index 0000000000..81885b255a --- /dev/null +++ b/changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md @@ -0,0 +1 @@ +- A voice connector (Plaud, WisprFlow) added, edited or removed from the Knowledge Integrations dashboard now shows up in, or leaves, the Inbox on the same save instead of after a restart, and the Inbox's direct Plaud read path is attached at boot too. When the saved connector list could not be applied to the running syncer, the save answers with `runtime_error` and the dashboard toasts `saved, but the running syncer did not reload: ...` instead of a plain success ([#18091](https://github.com/hivecommons/hive/issues/18091)). diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index d4a3e0808b..cd41274df9 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -6,8 +6,10 @@ import ( "path/filepath" "time" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" ) @@ -21,17 +23,7 @@ const inboxSweepTimeout = 2 * time.Minute // route state persists at <baseDir>/connector-state/_inbox.json (connector // names cannot start with "_", so it never collides with a connector). func (b *boot) bootInbox(baseDir string) { - sources := b.cfg.Inbox.EffectiveSources() - configured := map[string]bool{} - for _, c := range b.cfg.Knowledge.Connectors { - configured[c.Type] = true - } - var srcs []inbox.Source - for _, typ := range sources { - if configured[typ] { - srcs = append(srcs, b.inboxSource(baseDir, typ)) - } - } + srcs := b.inboxSources(baseDir, b.cfg.Knowledge.Connectors, b.knowledgeConnectors) token := func(ctx context.Context, _ string) (string, error) { if b.appAuth == nil { return "", nil @@ -58,18 +50,50 @@ func (b *boot) bootInbox(baseDir string) { b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) } +// inboxSources returns one source per configured voice type (inbox.sources) +// that entries provide. syncer, when non-nil, lets Plaud read recordings +// directly from its connector. +func (b *boot) inboxSources(baseDir string, entries []config.KnowledgeConnector, syncer *connector.Syncer) []inbox.Source { + configured := map[string]bool{} + for _, c := range entries { + configured[c.Type] = true + } + var srcs []inbox.Source + for _, typ := range b.cfg.Inbox.EffectiveSources() { + if configured[typ] { + srcs = append(srcs, b.inboxSource(baseDir, typ, entries, syncer)) + } + } + return srcs +} + +// refreshInboxSources rebuilds the Inbox sources from the live connector set +// (#18091). bootKnowledgeConnectors runs it once the boot syncer exists (the +// Inbox boots first, so its Plaud source would otherwise stay fact-only), and +// reloadKnowledgeConnectors runs it after every dashboard save that reached +// the runtime, so a voice connector added after boot feeds GET /api/inbox +// without a restart. A nil syncer keeps the fact-backed sources. +func (b *boot) refreshInboxSources(entries []config.KnowledgeConnector, syncer *connector.Syncer) { + if b.inboxSvc == nil { + return + } + srcs := b.inboxSources(knowledge.BaseDir(), entries, syncer) + b.inboxSvc.SetSources(srcs) + b.logger.Info("inbox sources refreshed", "sources", len(srcs)) +} + // inboxSource returns the fact-backed source for typ. For Plaud it prefers // reading recordings directly from the connector while it is signed in. -func (b *boot) inboxSource(baseDir, typ string) inbox.Source { +func (b *boot) inboxSource(baseDir, typ string, entries []config.KnowledgeConnector, syncer *connector.Syncer) inbox.Source { facts := &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ} - if typ != inbox.PlaudType || b.knowledgeConnectors == nil { + if typ != inbox.PlaudType || syncer == nil { return facts } - for _, c := range b.cfg.Knowledge.Connectors { + for _, c := range entries { if c.Type != typ { continue } - conn, ok := b.knowledgeConnectors.Connector(c.Name) + conn, ok := syncer.Connector(c.Name) reader, isReader := conn.(connector.PlaudReader) if !ok || !isReader { continue diff --git a/src/cmd/hive/boot_inbox_sources_test.go b/src/cmd/hive/boot_inbox_sources_test.go new file mode 100644 index 0000000000..f941408b14 --- /dev/null +++ b/src/cmd/hive/boot_inbox_sources_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +func plaudEntry() config.KnowledgeConnector { + return config.KnowledgeConnector{Name: "voice", Type: connector.TypePlaud, Layer: "personal"} +} + +// The Inbox follows the connector set the dashboard saves (#18091): a hive +// booted with no connectors lists no sources, a hot-loaded Plaud connector +// becomes the direct-read source on the same save, and removing every +// connector drops it again without a restart. +func TestReloadKnowledgeConnectorsRefreshesInboxSources(t *testing.T) { + knowledge.SetBaseDirForTest(t, t.TempDir()) + b, log := newDepsTestBoot(t, &config.Config{}) + b.bootKnowledgeConnectors(bootKnowledgeDeps{startConnectorSyncer: func(context.Context, *connector.Syncer) {}}) + if b.inboxSvc == nil { + t.Fatal("the Inbox must boot even with no connectors") + } + if b.inboxSvc.HasSource(inbox.PlaudType) { + t.Fatal("no connectors at boot must leave the Inbox without a plaud source") + } + + rt, err := b.reloadKnowledgeConnectors([]config.KnowledgeConnector{plaudEntry()}) + if err != nil || rt == nil { + t.Fatalf("reload = %v, %v\n%s", rt, err, log.String()) + } + if !b.inboxSvc.HasSource(inbox.PlaudType) { + t.Fatalf("hot-loaded plaud connector must become an Inbox source\n%s", log.String()) + } + srcs := b.inboxSources(knowledge.BaseDir(), []config.KnowledgeConnector{plaudEntry()}, b.knowledgeConnectors) + if len(srcs) != 1 { + t.Fatalf("sources = %d, want the one plaud source", len(srcs)) + } + if _, direct := srcs[0].(*inbox.PreferDirect); !direct { + t.Fatalf("plaud source with a live syncer = %T, want *inbox.PreferDirect", srcs[0]) + } + + if rt, err := b.reloadKnowledgeConnectors(nil); err != nil || rt == nil { + t.Fatalf("reload with none = %v, %v; want the same runtime kept", rt, err) + } + if b.inboxSvc.HasSource(inbox.PlaudType) { + t.Fatal("removing every connector must drop the Inbox plaud source") + } +} + +// A hive booted with a Plaud connector attaches the direct-read source once +// the syncer exists, not the fact-only one the Inbox had to settle for while +// booting ahead of the syncer (#18091). +func TestBootKnowledgeConnectorsAttachesDirectPlaudSource(t *testing.T) { + knowledge.SetBaseDirForTest(t, t.TempDir()) + cfg := &config.Config{} + cfg.Knowledge.Connectors = []config.KnowledgeConnector{plaudEntry()} + b, log := newDepsTestBoot(t, cfg) + + if srcs := b.inboxSources(knowledge.BaseDir(), cfg.Knowledge.Connectors, nil); len(srcs) != 1 { + t.Fatalf("sources without a syncer = %d, want one fact-backed plaud source", len(srcs)) + } else if _, facts := srcs[0].(*inbox.FactSource); !facts { + t.Fatalf("plaud source without a syncer = %T, want *inbox.FactSource", srcs[0]) + } + + b.bootKnowledgeConnectors(bootKnowledgeDeps{startConnectorSyncer: func(context.Context, *connector.Syncer) {}}) + if b.knowledgeConnectorRuntime() == nil { + t.Fatalf("syncer not built\n%s", log.String()) + } + if !b.inboxSvc.HasSource(inbox.PlaudType) { + t.Fatal("plaud source missing after boot") + } + srcs := b.inboxSources(knowledge.BaseDir(), cfg.Knowledge.Connectors, b.knowledgeConnectors) + if len(srcs) != 1 { + t.Fatalf("sources = %d, want one", len(srcs)) + } + if _, direct := srcs[0].(*inbox.PreferDirect); !direct { + t.Fatalf("plaud source after boot = %T, want *inbox.PreferDirect", srcs[0]) + } +} diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index de04a6344f..9b71766598 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -105,6 +105,9 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { b.knowledgeConnectorsMu.Lock() b.knowledgeConnectors = s b.knowledgeConnectorsMu.Unlock() + // The Inbox booted before the syncer existed; give it the live + // connectors so Plaud can read recordings directly (#18091). + b.refreshInboxSources(b.cfg.Knowledge.Connectors, s) if deps.startConnectorSyncer != nil { deps.startConnectorSyncer(b.ctx, s) } @@ -117,7 +120,9 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { // starts one from the same inputs as bootKnowledgeConnectors; otherwise it // reconfigures the running syncer in place, so unchanged connectors keep // syncing undisturbed. The knowledge.publish mirror is not rebuilt: a -// changed publish connector still needs a restart. +// changed publish connector still needs a restart. Once the runtime follows, +// the Inbox sources are rebuilt from entries so a voice connector added from +// the dashboard shows up in the Inbox at once. func (b *boot) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) (dashboard.KnowledgeConnectorRuntime, error) { b.knowledgeConnectorsMu.Lock() defer b.knowledgeConnectorsMu.Unlock() @@ -126,10 +131,16 @@ func (b *boot) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) (d return nil, err } b.logger.Info("knowledge connectors reloaded", "count", len(entries)) + b.refreshInboxSources(entries, b.knowledgeConnectors) return b.knowledgeConnectors, nil } s, err := newKnowledgeConnectorSyncer(entries, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil || s == nil { + // No runtime to follow: with no entries left the Inbox still lists + // no sources, and a build failure keeps the previous set. + if err == nil { + b.refreshInboxSources(entries, nil) + } return nil, err } b.knowledgeConnectors = s @@ -137,6 +148,7 @@ func (b *boot) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) (d b.startConnectorSyncer(b.ctx, s) } b.logger.Info("knowledge connectors started", "count", len(entries), "hot_loaded", true) + b.refreshInboxSources(entries, s) return s, nil } diff --git a/src/pkg/dashboard/api_config_knowledge_connectors.go b/src/pkg/dashboard/api_config_knowledge_connectors.go index 0471de50b4..d7c78c1106 100644 --- a/src/pkg/dashboard/api_config_knowledge_connectors.go +++ b/src/pkg/dashboard/api_config_knowledge_connectors.go @@ -264,14 +264,23 @@ func (s *Server) handleKnowledgeConnectorsPut(w http.ResponseWriter, r *http.Req if err := s.saveConfig(); err != nil { s.logger.Error("failed to persist config after knowledge connectors update", "error", err) } - s.reloadKnowledgeConnectors(next) + reloadErr := s.reloadKnowledgeConnectors(next) names := make([]string, 0, len(next)) for _, c := range next { names = append(names, c.Name) } s.auditFromRequest(r, "config_knowledge_connectors", auditDetail("connectors", strings.Join(names, ",")), "") s.refreshAndPersist() - jsonResponse(w, knowledgeConnectorsResponse(s.deps.Config)) + // The config is persisted even when the running syncer did not follow; + // runtime_error carries that verbatim so the dashboard can say "saved, + // but not reloaded" instead of a plain success (#18091). runtime keeps + // the status endpoint's meaning: a connector syncer is running. + resp := knowledgeConnectorsResponse(s.deps.Config) + if reloadErr != nil { + resp["runtime_error"] = reloadErr.Error() + } + resp["runtime"] = s.knowledgeConnectors() != nil + jsonResponse(w, resp) } // knowledgeConnectorPill maps a connector's runtime state to ok / syncing / diff --git a/src/pkg/dashboard/api_knowledge_connectors.go b/src/pkg/dashboard/api_knowledge_connectors.go index 481f69f8b1..d1091f0951 100644 --- a/src/pkg/dashboard/api_knowledge_connectors.go +++ b/src/pkg/dashboard/api_knowledge_connectors.go @@ -106,17 +106,20 @@ func (s *Server) setKnowledgeConnectors(rt KnowledgeConnectorRuntime) { // reloadKnowledgeConnectors hot-loads entries into the connector runtime // after a dashboard save (#18091), so a connector added, edited, enabled, // disabled or removed takes effect without restarting the hive. A failure -// is logged and leaves the previous runtime in place. -func (s *Server) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) { +// is logged, leaves the previous runtime in place and is returned so the +// save response can report it as runtime_error: the config is already +// persisted, so the caller must learn that the running hive did not follow. +func (s *Server) reloadKnowledgeConnectors(entries []config.KnowledgeConnector) error { if s.deps.ReloadKnowledgeConnectors == nil { - return + return nil } rt, err := s.deps.ReloadKnowledgeConnectors(entries) if err != nil { s.logger.Warn("knowledge connectors not reloaded; restart the hive to apply", "error", err) - return + return err } s.setKnowledgeConnectors(rt) + return nil } // knowledgeConnectorStatus finds one connector's status by name. diff --git a/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go b/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go index 9401dd37e6..b319635b56 100644 --- a/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go +++ b/src/pkg/dashboard/api_knowledge_connectors_hotload_test.go @@ -2,6 +2,7 @@ package dashboard import ( "encoding/json" + "errors" "net/http" "strconv" "testing" @@ -119,3 +120,57 @@ func TestKnowledgeConnectorsHotLoad(t *testing.T) { t.Fatalf("sync after remove = %d, want 404", rec.Code) } } + +// A save the running syncer refuses is still persisted, but the PUT response +// carries runtime_error verbatim and keeps the previous runtime, so the +// dashboard can say "saved, but not reloaded" instead of a plain success +// (#18091). A save that reloads cleanly answers without runtime_error. +func TestKnowledgeConnectorsPutReportsRuntimeError(t *testing.T) { + s := covApiServer(t) + prev := newTestConnectorSyncer(t, &blockingConn{}) + s.deps.KnowledgeConnectors = prev + var fail bool + s.deps.ReloadKnowledgeConnectors = func([]config.KnowledgeConnector) (KnowledgeConnectorRuntime, error) { + if fail { + return nil, errors.New("plaud: scope.home must be an absolute path") + } + return prev, nil + } + body := `{"connectors":[{"name":"docs","type":"document","enabled":true,"layer":"project","scope":{"url":"https://example.org/guide.html"}}]}` + + put := func() map[string]any { + t.Helper() + rec := doPutRaw(s, "/api/config/knowledge/connectors", body) + if rec.Code != http.StatusOK { + t.Fatalf("PUT = %d %q", rec.Code, rec.Body.String()) + } + var resp map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v (%s)", err, rec.Body.String()) + } + return resp + } + + resp := put() + if _, ok := resp["runtime_error"]; ok { + t.Fatalf("clean reload must not report runtime_error: %v", resp) + } + if resp["runtime"] != true { + t.Fatalf("runtime = %v, want true while a syncer is running", resp["runtime"]) + } + + fail = true + resp = put() + if got, _ := resp["runtime_error"].(string); got != "plaud: scope.home must be an absolute path" { + t.Fatalf("runtime_error = %q, want the reload error verbatim (%v)", got, resp) + } + if resp["runtime"] != true { + t.Fatalf("runtime = %v, want true: the previous syncer stays in place", resp["runtime"]) + } + if s.knowledgeConnectors() != prev { + t.Fatal("a failed reload must keep the previous runtime") + } + if got := s.deps.Config.Knowledge.Connectors; len(got) != 1 || got[0].Name != "docs" { + t.Fatalf("config after failed reload = %+v, want the saved list", got) + } +} diff --git a/src/pkg/dashboard/knowledge_connector_reload_error.test.js b/src/pkg/dashboard/knowledge_connector_reload_error.test.js new file mode 100644 index 0000000000..d0dd49cc3a --- /dev/null +++ b/src/pkg/dashboard/knowledge_connector_reload_error.test.js @@ -0,0 +1,83 @@ +// kcSave surfaces a connector save the running syncer did not follow +// (hivecommons/hive#18091): the PUT answers 200 with `runtime_error` because +// the config is already persisted, and kcSave rejects with a message that +// says so, so every caller's catch handler toasts it instead of the plain +// "saved" success. A response without runtime_error resolves as before. +// Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +const ASYNC_PREFIX = 'async '; +const fnSrc = name => { + let start = html.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + if (html.slice(start - ASYNC_PREFIX.length, start) === ASYNC_PREFIX) start -= ASYNC_PREFIX.length; + let depth = 0; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) return html.slice(start, i + 1); + } + throw new Error('unbalanced ' + name); +}; + +const HTTP_OK = 200; +const HTTP_BAD_REQUEST = 400; +const RELOAD_ERROR = 'plaud: scope.home must be an absolute path'; +const SAVED_LIST = [{ name: 'voice', type: 'plaud', enabled: true }]; + +// Drive the real kcSave with a stubbed fetch; returns the state it mutated +// and how often it refreshed the list so the saved path is still observable. +const save = async (res, list = SAVED_LIST) => { + const calls = { reloads: 0, puts: [] }; + const fetch = async (url, init) => { calls.puts.push({ url, init }); return res; }; + const state = { connectors: [] }; + const run = new Function('fetch', '_kcState', 'loadKnowledgeConnectors', + fnSrc('kcSave') + '\nreturn kcSave;'); + const kcSave = run(fetch, state, async () => { calls.reloads++; }); + const outcome = await kcSave(list).then(() => null, e => e); + return { outcome, state, calls }; +}; + +test('runtime_error on a 200 rejects with the saved-but-not-reloaded message, after the list refreshed', async () => { + const res = { ok: true, status: HTTP_OK, json: async () => ({ connectors: SAVED_LIST, runtime: true, runtime_error: RELOAD_ERROR }) }; + const { outcome, state, calls } = await save(res); + assert.ok(outcome instanceof Error, 'kcSave must reject'); + assert.strictEqual(outcome.message, 'saved, but the running syncer did not reload: ' + RELOAD_ERROR); + assert.deepStrictEqual(state.connectors, SAVED_LIST, 'the persisted list is still stored'); + assert.strictEqual(calls.reloads, 1, 'the connector list is still reloaded before the error surfaces'); +}); + +test('the rejection reaches the caller toast with the reload error verbatim', async () => { + const res = { ok: true, status: HTTP_OK, json: async () => ({ connectors: SAVED_LIST, runtime_error: RELOAD_ERROR }) }; + const { outcome } = await save(res); + const toasts = []; + const showToast = (msg, kind) => toasts.push({ msg, kind }); + // Mirrors kcToggleEnabled's handlers: the catch branch fires, not the success toast. + await Promise.reject(outcome).then(() => showToast('Connector voice updated', 'success')).catch(e => showToast('Connector update failed: ' + e.message, 'error')); + assert.deepStrictEqual(toasts, [{ msg: 'Connector update failed: saved, but the running syncer did not reload: ' + RELOAD_ERROR, kind: 'error' }]); +}); + +test('a clean save without runtime_error resolves', async () => { + const res = { ok: true, status: HTTP_OK, json: async () => ({ connectors: SAVED_LIST, runtime: true }) }; + const { outcome, state, calls } = await save(res); + assert.strictEqual(outcome, null); + assert.deepStrictEqual(state.connectors, SAVED_LIST); + assert.strictEqual(calls.reloads, 1); +}); + +test('an empty runtime_error is not an error', async () => { + const res = { ok: true, status: HTTP_OK, json: async () => ({ connectors: SAVED_LIST, runtime_error: '' }) }; + const { outcome } = await save(res); + assert.strictEqual(outcome, null); +}); + +test('a rejected save still reports the server error and does not refresh the list', async () => { + const res = { ok: false, status: HTTP_BAD_REQUEST, json: async () => ({ error: 'name is required' }) }; + const { outcome, calls } = await save(res); + assert.ok(outcome instanceof Error); + assert.strictEqual(outcome.message, 'name is required'); + assert.strictEqual(calls.reloads, 0); +}); diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index bdfdb07771..28de8d4a1e 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -694,6 +694,9 @@ if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); _kcState.connectors = data.connectors || []; await loadKnowledgeConnectors(); + // The config is saved even when the running syncer did not follow; + // surface that verbatim instead of the caller's success toast (#18091). + if (data.runtime_error) throw new Error('saved, but the running syncer did not reload: ' + data.runtime_error); } function kcFind(name) { return _kcState.connectors.find(c => c.name === name); } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 1646e1c9a0..bb6b943363 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -57941,6 +57941,9 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); _kcState.connectors = data.connectors || []; await loadKnowledgeConnectors(); + // The config is saved even when the running syncer did not follow; + // surface that verbatim instead of the caller's success toast (#18091). + if (data.runtime_error) throw new Error('saved, but the running syncer did not reload: ' + data.runtime_error); } function kcFind(name) { return _kcState.connectors.find(c => c.name === name); } diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go index ff1bcc3c92..35f6c4c5cd 100644 --- a/src/pkg/inbox/service.go +++ b/src/pkg/inbox/service.go @@ -64,6 +64,24 @@ func NewService(cfg config.InboxConfig, store *Store, sources []Source, dests [] return s } +// SetSources replaces the sources, keyed by type (#18091): a voice connector +// added through the dashboard becomes an inbox source without a restart. +// Items and Sweep pick the new set up on their next call. +func (s *Service) SetSources(sources []Source) { + next := make(map[string]Source, len(sources)) + for _, src := range sources { + next[src.Type()] = src + } + s.mu.Lock() + s.sources = next + s.mu.Unlock() +} + +// HasSource reports whether a source of typ is wired. +func (s *Service) HasSource(typ string) bool { + return s.source(typ) != nil +} + // SetReadiness installs the probes behind Settings.SpekReady / DibsTokenSet. func (s *Service) SetReadiness(spek, dibs func() bool) { s.spekReady, s.dibsTokenSet = spek, dibs diff --git a/src/pkg/inbox/service_sources_test.go b/src/pkg/inbox/service_sources_test.go new file mode 100644 index 0000000000..6f152ed889 --- /dev/null +++ b/src/pkg/inbox/service_sources_test.go @@ -0,0 +1,42 @@ +package inbox + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// TestSetSourcesFollowsConnectors: a service booted with no sources lists +// nothing, picks up a source handed to SetSources on the next Items call, +// and drops it again when the set is replaced (#18091). +func TestSetSourcesFollowsConnectors(t *testing.T) { + store := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + svc := NewService(config.InboxConfig{}, store, nil, nil, slog.New(slog.NewTextHandler(os.Stderr, nil))) + if svc.HasSource(PlaudType) { + t.Fatal("no sources wired at construction") + } + if items, err := svc.Items(context.Background()); err != nil || len(items) != 0 { + t.Fatalf("Items without sources = %+v, %v", items, err) + } + + svc.SetSources([]Source{&fakeSource{items: []Item{{Source: PlaudType, Connector: "voice", ID: "r1", Title: "Standup"}}}}) + if !svc.HasSource(PlaudType) { + t.Fatal("SetSources must wire the plaud source") + } + items, err := svc.Items(context.Background()) + if err != nil || len(items) != 1 || items[0].ID != "r1" { + t.Fatalf("Items after SetSources = %+v, %v", items, err) + } + + svc.SetSources(nil) + if svc.HasSource(PlaudType) { + t.Fatal("SetSources(nil) must drop every source") + } + if items, err := svc.Items(context.Background()); err != nil || len(items) != 0 { + t.Fatalf("Items after clearing sources = %+v, %v", items, err) + } +} From ed057e63b2b7ed64559faedf64b9b4dae40e9f26 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 23:08:09 +0000 Subject: [PATCH 180/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.248.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.248.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/added-17975-review-perspectives-source.md | 1 - ...dded-18091-connector-hotload-inbox-and-reload-errors.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-17975-review-perspectives-source.md delete mode 100644 changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2006a9c374..e6903dcf80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.248.0) + +### Added + +- **`review.perspectives_source` is documented.** `src/docs/review-swarm.md` now describes linking the review perspective set to a `kind: ReviewPerspectives` file in a GitHub repo (names and focus text only; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay operator-owned), the source-wins/inline-fallback rule, the last-known-good fallback order with a warning on a malformed document, the shared seed-only `github_prompt_allowlist` trust gate, and the read-only Review Gate display. `hive.yaml.example` gains a commented `perspectives_source:` block ([#17975](https://github.com/hivecommons/hive/issues/17975), [#17990](https://github.com/hivecommons/hive/issues/17990)). +- A voice connector (Plaud, WisprFlow) added, edited or removed from the Knowledge Integrations dashboard now shows up in, or leaves, the Inbox on the same save instead of after a restart, and the Inbox's direct Plaud read path is attached at boot too. When the saved connector list could not be applied to the running syncer, the save answers with `runtime_error` and the dashboard toasts `saved, but the running syncer did not reload: ...` instead of a plain success ([#18091](https://github.com/hivecommons/hive/issues/18091)). + ## 2026-11-13 (v5.247.2) ### Fixed diff --git a/changelog.d/added-17975-review-perspectives-source.md b/changelog.d/added-17975-review-perspectives-source.md deleted file mode 100644 index caffb45297..0000000000 --- a/changelog.d/added-17975-review-perspectives-source.md +++ /dev/null @@ -1 +0,0 @@ -- **`review.perspectives_source` is documented.** `src/docs/review-swarm.md` now describes linking the review perspective set to a `kind: ReviewPerspectives` file in a GitHub repo (names and focus text only; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay operator-owned), the source-wins/inline-fallback rule, the last-known-good fallback order with a warning on a malformed document, the shared seed-only `github_prompt_allowlist` trust gate, and the read-only Review Gate display. `hive.yaml.example` gains a commented `perspectives_source:` block ([#17975](https://github.com/hivecommons/hive/issues/17975), [#17990](https://github.com/hivecommons/hive/issues/17990)). diff --git a/changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md b/changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md deleted file mode 100644 index 81885b255a..0000000000 --- a/changelog.d/added-18091-connector-hotload-inbox-and-reload-errors.md +++ /dev/null @@ -1 +0,0 @@ -- A voice connector (Plaud, WisprFlow) added, edited or removed from the Knowledge Integrations dashboard now shows up in, or leaves, the Inbox on the same save instead of after a restart, and the Inbox's direct Plaud read path is attached at boot too. When the saved connector list could not be applied to the running syncer, the save answers with `runtime_error` and the dashboard toasts `saved, but the running syncer did not reload: ...` instead of a plain success ([#18091](https://github.com/hivecommons/hive/issues/18091)). From 5eef80d3eb319f18b143aa2d159f6ca1784a7da4 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 18:16:51 -0500 Subject: [PATCH 181/223] =?UTF-8?q?=F0=9F=94=92=20fix(inference):=20refuse?= =?UTF-8?q?=20proxy-UID=20callers=20that=20claim=20a=20UID-isolated=20agen?= =?UTF-8?q?t's=20key=20(#18133)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔒 fix(inference): refuse proxy-UID callers that claim a UID-isolated agent's key On v5 pods the hive drops to dev (1001), which is the UID map's ProxyUID, and every agent without a per-agent UID runs as that same user. Since #18090 the translator trusts the self-asserted sk-hive-<name> header from that UID while the private-forward guard is in place, so one UID-less agent could claim another agent's name and borrow its inference route, upstream credential and usage attribution (#18115). A caller at the proxy UID may now only assert a name that has no per-agent UID. An agent that owns a UID always connects from that UID, so a proxy-UID socket naming it is refused with reason=proxy_uid_claims_isolated_agent before any route lookup. Names with no UID (the #18061 shape) are still accepted, root is still the trusted control plane, and HIVE_PROXY_ADVISORY_OK keeps its documented meaning. Fixes #18115 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🔒 fix(inference): advisory-ok does not lift the isolated-agent spoof refusal HIVE_PROXY_ADVISORY_OK accepts a self-asserted name where the socket proves nothing about the caller. A proxy-UID socket claiming an agent that owns its own UID is a socket that proves the claim false, so the advisory escape hatch no longer overrides sharedUIDClaimsIsolatedAgent. Review follow-up on #18133. Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 fix(inference): scope the isolated-agent spoof check to the proxy UID The advisory-ok guard applied to every non-root UID, which refused the documented HIVE_PROXY_ADVISORY_OK path for an unmapped non-proxy UID (TestTranslatorAgentName_UnmappedRefusedUnlessAdvisoryOK). The shared identity #18115 is about is the proxy UID, so the check now applies only when the socket owner IS the proxy UID; other unmapped UIDs keep the advisory semantics unchanged. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18115-proxy-uid-identity-spoof.md | 1 + src/crux/proxy/github_proxy_inference.go | 14 ++++- src/crux/proxy/inference_refusal.go | 27 +++++++-- .../inference_translator_identity_test.go | 57 +++++++++++++++++++ 4 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-18115-proxy-uid-identity-spoof.md diff --git a/changelog.d/fixed-18115-proxy-uid-identity-spoof.md b/changelog.d/fixed-18115-proxy-uid-identity-spoof.md new file mode 100644 index 0000000000..6f6dec0214 --- /dev/null +++ b/changelog.d/fixed-18115-proxy-uid-identity-spoof.md @@ -0,0 +1 @@ +- The inference translator no longer lets a caller at the shared proxy UID (`dev`, 1001 on v5 pods, where the hive process and every agent without a per-agent UID run) claim the `sk-hive-<name>` key of an agent that has its own UID: such an agent always connects from that UID, so the claim is refused with `reason=proxy_uid_claims_isolated_agent` and the victim's route and upstream credential are never used. Names with no per-agent UID are still accepted from the proxy UID, root stays trusted, and `HIVE_PROXY_ADVISORY_OK` still accepts self-asserted identity only where the socket proves nothing, not when it proves the claim false (#18115). diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 4d7333ded7..2a3f500ffb 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -58,7 +58,13 @@ const inferenceAPIKeyPrefix = "sk-hive-" // tunnel legs can relay an agent-controlled request to this loopback // port from the proxy's own socket; those legs dial only globally // routable addresses unless HIVE_PROXY_ALLOW_PRIVATE_FORWARD lifts the -// guard, so with the guard lifted the proxy UID is still refused; +// guard, so with the guard lifted the proxy UID is still refused. Even +// with the guard in place the proxy UID is a SHARED identity: every +// UID-less agent runs as it, so one such agent could claim another +// agent's key. A caller at that UID may therefore only assert a name +// that has no per-agent UID: an agent that owns a UID always arrives +// from that UID, so a proxy-UID socket claiming it is refused, and +// HIVE_PROXY_ADVISORY_OK does not lift that refusal (#18115); // - an unmapped UID, or no UID map at all, accepts the header only under // HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path // honours for deployments without per-agent UID separation. @@ -82,11 +88,13 @@ func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { } return name, true } - if p.internalTranslatorUID(uid) { + if p.internalTranslatorUID(uid) && !p.sharedUIDClaimsIsolatedAgent(uid, claimed) { return claimed, true } } - if p.proxyAdvisoryOK { + // Advisory-ok accepts self-asserted identity where the socket proves + // nothing; it does not override a socket that proves the claim false. + if p.proxyAdvisoryOK && !(err == nil && p.sharedUIDClaimsIsolatedAgent(uid, claimed)) { return claimed, true } return "", false diff --git a/src/crux/proxy/inference_refusal.go b/src/crux/proxy/inference_refusal.go index e797f17cd1..747c0d96f6 100644 --- a/src/crux/proxy/inference_refusal.go +++ b/src/crux/proxy/inference_refusal.go @@ -1,6 +1,9 @@ package proxy -import "net/http" +import ( + "net/http" + "strings" +) // internalTranslatorUID reports whether uid is the hive's own control plane // for the translator's purposes: root always, and the proxy UID only when the @@ -12,12 +15,25 @@ func (p *GitHubProxy) internalTranslatorUID(uid int) bool { return uid == 0 || !p.allowPrivateForward } +// sharedUIDClaimsIsolatedAgent reports whether a caller at the proxy UID is +// asserting the name of an agent that has its own per-agent UID. The proxy +// UID is shared by the hive process and every UID-less agent, so the header +// is the only thing distinguishing those callers from each other; an agent +// that owns a UID never connects from the proxy UID, so such a claim can only +// be another process borrowing that agent's inference route (#18115). Root is +// the control plane and is not subject to this check, and other unmapped UIDs +// keep the HIVE_PROXY_ADVISORY_OK semantics the GitHub path documents. +func (p *GitHubProxy) sharedUIDClaimsIsolatedAgent(uid int, claimed string) bool { + return uid != 0 && uid == p.uidMap.ProxyUID && claimed != "" && p.uidMap.LookupByName(claimed) > 0 +} + // translatorRefusalAttrs are the log attributes for a refused translator // request. Beyond remote/method/path they say why identification failed (no UID // map, socket not resolvable, proxy UID refused because private forwarding is -// enabled, or UID not mapped to an agent), the socket owner's -// UID when known, and whether an x-api-key was sent, so the refusal is -// actionable without weakening it. The key value is never logged. +// enabled, proxy UID claiming a UID-isolated agent's name, or UID not mapped +// to an agent), the socket owner's UID when known, and whether an x-api-key +// was sent, so the refusal is actionable without weakening it. The key value +// is never logged; the claimed agent NAME is (it is public inside the pod). func (p *GitHubProxy) translatorRefusalAttrs(r *http.Request) []any { attrs := []any{"remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path} switch { @@ -25,10 +41,13 @@ func (p *GitHubProxy) translatorRefusalAttrs(r *http.Request) []any { attrs = append(attrs, "reason", "no_uid_map") default: uid, err := uidForRemoteAddr(r.RemoteAddr) + claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) if err != nil { attrs = append(attrs, "reason", "socket_owner_unresolved", "error", err.Error()) } else if p.uidMap.IsInternalUID(uid) && p.allowPrivateForward { attrs = append(attrs, "reason", "proxy_uid_private_forward", "uid", uid) + } else if p.uidMap.IsInternalUID(uid) && p.sharedUIDClaimsIsolatedAgent(uid, claimed) { + attrs = append(attrs, "reason", "proxy_uid_claims_isolated_agent", "uid", uid, "claimed", claimed) } else { attrs = append(attrs, "reason", "uid_not_mapped", "uid", uid) } diff --git a/src/crux/proxy/inference_translator_identity_test.go b/src/crux/proxy/inference_translator_identity_test.go index d34e3ea640..726b38dbdd 100644 --- a/src/crux/proxy/inference_translator_identity_test.go +++ b/src/crux/proxy/inference_translator_identity_test.go @@ -141,6 +141,63 @@ func TestTranslatorAgentName_ProxyUIDTrustedOnlyWhileForwardGuarded(t *testing.T } } +// TestTranslatorAgentName_ProxyUIDCannotClaimIsolatedAgent pins #18115: the +// proxy UID is shared by the hive process and every UID-less agent, so a +// caller at that UID may only assert a name that has NO per-agent UID. A +// claim on an agent that owns a UID (which would have arrived from that UID) +// is refused, whatever the private-forward setting; root is still trusted. +func TestTranslatorAgentName_ProxyUIDCannotClaimIsolatedAgent(t *testing.T) { + fakeProcNetTCP(t) + var buf strings.Builder + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(&buf, nil)), inference: newInferenceRouter()} + p.uidMap = &agent.UIDMap{BaseUID: 2001, ProxyUID: 2003, Agents: map[string]int{"quality": 2002}} + // The victim has a real route; the spoofed request must never reach it. + p.inference.Set("quality", &InferenceRoute{Backend: "vllm", Endpoint: "http://127.0.0.1:1", Model: "m", APIKey: "victim-secret"}) + + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); ok || name != "" { + t.Fatalf("proxy uid claiming isolated agent: got (%q,%v), want refused", name, ok) + } + + rec := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(rec, translatorReq("127.0.0.1:43210", "quality")) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401; body %s", rec.Code, rec.Body.String()) + } + for _, want := range []string{"reason=proxy_uid_claims_isolated_agent", "uid=2003", "claimed=quality"} { + if !strings.Contains(buf.String(), want) { + t.Errorf("refusal log %q missing %q", buf.String(), want) + } + } + for _, leak := range []string{"inference forward", "victim-secret", "sk-hive-"} { + if strings.Contains(buf.String(), leak) { + t.Errorf("log %q contains %q: the victim's route was used or the key leaked", buf.String(), leak) + } + } + + // A name with no per-agent UID is the legitimate shape of a UID-less + // agent (#18061) and is still accepted from the proxy UID. + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "supervisor" { + t.Fatalf("proxy uid claiming uid-less agent: got (%q,%v), want (supervisor,true)", name, ok) + } + + // HIVE_PROXY_ADVISORY_OK accepts self-asserted identity only where the + // socket proves nothing; a socket that proves the claim false still wins. + p.proxyAdvisoryOK = true + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); ok || name != "" { + t.Fatalf("advisory-ok proxy uid claiming isolated agent: got (%q,%v), want refused", name, ok) + } + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "supervisor" { + t.Fatalf("advisory-ok proxy uid claiming uid-less agent: got (%q,%v), want (supervisor,true)", name, ok) + } + p.proxyAdvisoryOK = false + + // Root is the control plane that issues the keys: it may still name any agent. + fakeProcNetTCPUID(t, 0) + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); !ok || name != "quality" { + t.Fatalf("root claiming isolated agent: got (%q,%v), want (quality,true)", name, ok) + } +} + // TestTranslatorAgentName_NoUIDMap pins the no-map deployments: the header is // trusted only under HIVE_PROXY_ADVISORY_OK, matching fallbackAgentName. func TestTranslatorAgentName_NoUIDMap(t *testing.T) { From f60c245b739222c6540e3672ffd336ac5405a513 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 18:24:48 -0500 Subject: [PATCH 182/223] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20native-queue-?= =?UTF-8?q?policy=20names=20why=20a=20PR=20is=20ineligible=20instead=20of?= =?UTF-8?q?=20a=20generic=20verdict=20(#18134)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(ci): native-queue-policy names why a PR is ineligible instead of a generic verdict Draft, closed, held and head/base-moved PRs all failed with the same line, "head/base moved, draft, closed, or held". Five sibling PRs tripping it for independent per-PR reasons (needs-human, hold, draft + do-not-merge) read like a broken shared workflow and were filed as a shared-CI incident (#18128). The verdict was correct on each; only the message was opaque. assertEligible now reports the specific condition and states that it is PR state, not a CI failure. Fail-closed semantics are unchanged. Fixes #18128 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 test(ci): match the specific moved-head message in the evaluate test Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .github/scripts/native-queue-policy.cjs | 21 ++++++++++++++++--- .github/scripts/native-queue-policy.test.cjs | 15 ++++++++++++- .../fixed-18128-native-queue-policy-reason.md | 1 + 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-18128-native-queue-policy-reason.md diff --git a/.github/scripts/native-queue-policy.cjs b/.github/scripts/native-queue-policy.cjs index a54bcab393..9bd9b8df13 100644 --- a/.github/scripts/native-queue-policy.cjs +++ b/.github/scripts/native-queue-policy.cjs @@ -99,10 +99,25 @@ function membersForGroup(entries, group) { return members; } +// Name the condition that tripped: every one of these is a per-PR state the +// author or an operator must change, never a shared CI incident. An opaque +// "moved, draft, closed, or held" verdict across several held or draft PRs +// reads like a broken workflow and gets filed as one (#18128). +function ineligibleReason(pr, head, base) { + if (pr.state !== 'open') return `is ${pr.state}, not open`; + if (pr.draft) return 'is a draft'; + if (pr.head?.sha !== head) return `head moved to ${pr.head?.sha} (event head ${head})`; + if (pr.base?.ref !== base) return `base moved to ${pr.base?.ref} (event base ${base})`; + const held = (pr.labels ?? []).filter(l => BLOCKED_LABEL.test(l.name)).map(l => l.name); + if (held.length) return `is held by label ${held.join(', ')}`; + return null; +} + function assertEligible(pr, head, base) { - if (pr.state !== 'open' || pr.draft || pr.head?.sha !== head || - pr.base?.ref !== base || pr.labels?.some(l => BLOCKED_LABEL.test(l.name))) { - throw new Error(`#${pr.number}: head/base moved, draft, closed, or held`); + const reason = ineligibleReason(pr, head, base); + if (reason) { + throw new Error(`#${pr.number} ${reason}; not eligible for admission ` + + '(PR state, not a CI failure: re-evaluated on the next synchronize, ready_for_review or hold-label change)'); } } diff --git a/.github/scripts/native-queue-policy.test.cjs b/.github/scripts/native-queue-policy.test.cjs index 742d90ffa7..8e7d626d63 100644 --- a/.github/scripts/native-queue-policy.test.cjs +++ b/.github/scripts/native-queue-policy.test.cjs @@ -306,6 +306,19 @@ test('draft, closed, held and moved-head/base PRs cannot pass', () => { ]) assert.throws(() => assertEligible({ ...eligible(1), ...change }, sha(11), 'v5')); }); +test('ineligibility names the tripped condition, not a generic verdict (#18128)', () => { + const expect = (change, pattern) => + assert.throws(() => assertEligible({ ...eligible(1), ...change }, sha(11), 'v5'), pattern); + expect({ draft: true }, /#1 is a draft/); + expect({ state: 'closed' }, /#1 is closed, not open/); + expect({ head: { sha: sha(99) } }, new RegExp(`#1 head moved to ${sha(99)} \\(event head ${sha(11)}\\)`)); + expect({ base: { ref: 'v6' } }, /#1 base moved to v6 \(event base v5\)/); + expect({ labels: [{ name: 'size/L' }, { name: 'needs-human' }, { name: 'hold' }] }, + /#1 is held by label needs-human, hold/); + expect({ labels: [{ name: 'do-not-merge/work-in-progress' }] }, /PR state, not a CI failure/); + assert.doesNotThrow(() => assertEligible({ ...eligible(1), labels: [{ name: 'stage/070-fixing' }] }, sha(11), 'v5')); +}); + test('group evaluation checks each member head, all evidence, and live synthetic ref', async () => { const f = fixture(); assert.match(await evaluate(f), /all 2 member heads/); @@ -381,7 +394,7 @@ test('a 5xx from the API is retried as pending evidence, never read as a verdict test('open PR with moved head still fails closed on pull_request_target', async () => { const f = fixture('pull_request_target'); f.state.prs[0].head.sha = sha(99); - await assert.rejects(evaluate(f), /head\/base moved/); + await assert.rejects(evaluate(f), /#1 head moved to/); }); test('untrusted events and workflow provenance are rejected', async () => { diff --git a/changelog.d/fixed-18128-native-queue-policy-reason.md b/changelog.d/fixed-18128-native-queue-policy-reason.md new file mode 100644 index 0000000000..1773921570 --- /dev/null +++ b/changelog.d/fixed-18128-native-queue-policy-reason.md @@ -0,0 +1 @@ +- `native-queue-policy` now names the condition that made a PR ineligible for admission (`is a draft`, `is held by label needs-human`, `head moved to <sha>`, `base moved to v6`, `is closed`) instead of the single opaque `head/base moved, draft, closed, or held` line, and says it is PR state rather than a CI failure. Several held or draft PRs failing with the same generic text looked like a broken shared workflow and were filed as a shared-CI incident (#18128); the check's fail-closed verdict was correct on every one of them. From 184cc0f329f020a6cd917505df0fe6d7cf3c2e66 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 18:32:57 -0500 Subject: [PATCH 183/223] =?UTF-8?q?=E2=9C=A8=20Sessions=20backend:=20named?= =?UTF-8?q?,=20shared=20interactive=20CLI=20terminals=20(flag=20off=20by?= =?UTF-8?q?=20default)=20(#18101)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Sessions backend: named, shared interactive CLI terminals (flag off by default) Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions backend: fix decodeJSON redeclaration, config golden, settings parity, citation drift Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions tests: register the session routes on the bare test server Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions tests: use a valid session name Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions tests: drop the settings-parity allowlist entry for dashboard.sessions.enabled The key is already referenced from the dashboard package (deps.go), so TestSettingsDialogReferencesEveryHiveYAMLLeaf requires the allowlist entry to be removed. The other three dashboard.sessions.* limits stay allowlisted: nothing in the dashboard package or Settings UI references them. Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions backend: refresh API reference citations after rebase onto v5 Signed-off-by: Andy Anderson <andy@clubanderson.com> * ✅ Sessions: cover usersession error paths Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions: catch schemeless proxy credentials in the session env filter Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 Sessions: drop the now-unused net/url import Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- bin/ttyd-tmux.sh | 18 + changelog.d/added-sessions-backend.md | 1 + dashboard/openapi.json | 305 ++++++ src/cmd/hive/main.go | 1 + src/crux/agent/usersession.go | 380 ++++++++ src/crux/agent/usersession_env.go | 139 +++ src/crux/agent/usersession_home.go | 116 +++ src/crux/agent/usersession_more_test.go | 509 ++++++++++ src/crux/agent/usersession_test.go | 474 ++++++++++ src/crux/runtime/code_work.go | 11 + src/crux/usersession_types.go | 42 + src/deploy/ttyd-tmux.sh | 13 + src/docs/api-reference.md | 5 + src/docs/api-reference/agents-and-controls.md | 26 +- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/beads.md | 10 +- src/docs/api-reference/configuration.md | 38 +- src/docs/api-reference/dashboard-misc.md | 30 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/inception.md | 28 +- src/docs/api-reference/knowledge.md | 90 +- src/docs/api-reference/nous.md | 50 +- src/docs/api-reference/packs-and-acmm.md | 36 +- src/docs/api-reference/sessions.md | 28 + src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/env-vars.md | 1 + src/docs/security-model.md | 15 + src/hive.yaml.example | 5 + src/pkg/config/config.go | 2 + src/pkg/config/dashboard_sessions.go | 105 +++ src/pkg/config/dashboard_sessions_test.go | 67 ++ .../config/testdata/agent_runtime_golden.json | 3 +- src/pkg/config/validate.go | 3 + src/pkg/dashboard/api.go | 1 + src/pkg/dashboard/api_sessions.go | 723 ++++++++++++++ src/pkg/dashboard/api_sessions_store.go | 222 +++++ src/pkg/dashboard/api_sessions_test.go | 885 ++++++++++++++++++ src/pkg/dashboard/api_sessions_upload.go | 134 +++ src/pkg/dashboard/audit.go | 5 + src/pkg/dashboard/deps.go | 3 + src/pkg/dashboard/server.go | 7 + .../dashboard/settings_parity_allowlist.txt | 3 + src/pkg/dashboard/terminal_handoff.go | 29 + 43 files changed, 4423 insertions(+), 170 deletions(-) create mode 100644 changelog.d/added-sessions-backend.md create mode 100644 src/crux/agent/usersession.go create mode 100644 src/crux/agent/usersession_env.go create mode 100644 src/crux/agent/usersession_home.go create mode 100644 src/crux/agent/usersession_more_test.go create mode 100644 src/crux/agent/usersession_test.go create mode 100644 src/crux/usersession_types.go create mode 100644 src/docs/api-reference/sessions.md create mode 100644 src/pkg/config/dashboard_sessions.go create mode 100644 src/pkg/config/dashboard_sessions_test.go create mode 100644 src/pkg/dashboard/api_sessions.go create mode 100644 src/pkg/dashboard/api_sessions_store.go create mode 100644 src/pkg/dashboard/api_sessions_test.go create mode 100644 src/pkg/dashboard/api_sessions_upload.go diff --git a/bin/ttyd-tmux.sh b/bin/ttyd-tmux.sh index 36ce5a950f..70e219a48e 100755 --- a/bin/ttyd-tmux.sh +++ b/bin/ttyd-tmux.sh @@ -30,6 +30,24 @@ else SESSION="supervisor" SESSION_FROM_ARG=0 fi +# User sessions (dashboard Sessions feature): hive-sess-<id> runs on its OWN +# tmux server whose socket is named after the session. Reject any other +# hive-sess-* shape so ?arg= cannot reach a glob or a path. +USER_SESSION_RE='^hive-sess-[a-f0-9]{8,32}$' +case "$SESSION" in + hive-sess-*) + if ! [[ "$SESSION" =~ $USER_SESSION_RE ]]; then + echo "error: invalid user session name" >&2 + exit 1 + fi + ;; +esac +case "$SESSION" in + hive-sess-*) + # Per-session tmux server: every tmux call below targets its socket. + tmux() { command tmux -L "$SESSION" "$@"; } + ;; +esac TTYD_WAIT_SEC="${HIVE_TTYD_WAIT_SEC:-900}" TTYD_POLL_SEC="${HIVE_TTYD_POLL_SEC:-2}" TTYD_HISTORY_LIMIT="${HIVE_TTYD_HISTORY_LIMIT:-50000}" diff --git a/changelog.d/added-sessions-backend.md b/changelog.d/added-sessions-backend.md new file mode 100644 index 0000000000..fc9173aaab --- /dev/null +++ b/changelog.d/added-sessions-backend.md @@ -0,0 +1 @@ +- Dashboard Sessions backend (off by default, `dashboard.sessions.enabled` / `HIVE_SESSIONS_ENABLED`): named, shared, interactive CLI terminals behind `/api/sessions` (create, rename, close, resume, scrollback, input and upload), with per-session HOME and a credential-free environment, no permission-bypass flags, per-spoke and per-user limits, an idle sweeper and `session_*` audit entries; `/api/terminal/handoff` accepts `{"session":"<id>"}`. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 14efdd4274..78c8ff9081 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -83,6 +83,10 @@ { "name": "Swarm", "description": "Repo swarm mode status, lifecycle, and history" + }, + { + "name": "Sessions", + "description": "Interactive named CLI terminals" } ], "paths": { @@ -38344,6 +38348,307 @@ } } } + }, + "/api/sessions": { + "get": { + "tags": [ + "Sessions" + ], + "summary": "List sessions, vendors and limits", + "description": "Any role. While the feature is off returns enabled=false with empty lists.", + "responses": { + "200": { + "description": "OK" + } + } + }, + "post": { + "tags": [ + "Sessions" + ], + "summary": "Create a session (owner or read-write)", + "description": "Starts a named CLI session. Vendor must be a known, allowed, installed backend; never a shell, never a bypass-permissions launch.", + "responses": { + "201": { + "description": "OK" + }, + "400": { + "description": "Invalid name, group or vendor" + }, + "403": { + "description": "Role or vendor not allowed" + }, + "404": { + "description": "Sessions disabled" + }, + "409": { + "description": "Vendor not installed" + }, + "429": { + "description": "Session limit reached" + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "group": { + "type": "string" + }, + "vendor": { + "type": "string" + } + } + } + } + } + } + } + }, + "/api/sessions/{id}": { + "patch": { + "tags": [ + "Sessions" + ], + "summary": "Rename or regroup a session (creator or owner)", + "description": "Changes name and/or group.", + "responses": { + "200": { + "description": "OK" + }, + "403": { + "description": "Not the creator or an owner" + }, + "404": { + "description": "Unknown session" + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "group": { + "type": "string" + } + } + } + } + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "tags": [ + "Sessions" + ], + "summary": "Close a session (creator or owner)", + "description": "Kills tmux and keeps the metadata marked ended; on an ended session removes its metadata.", + "responses": { + "200": { + "description": "OK" + }, + "403": { + "description": "Not the creator or an owner" + }, + "404": { + "description": "Unknown session" + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/sessions/{id}/log": { + "get": { + "tags": [ + "Sessions" + ], + "summary": "Redacted scrollback snapshot (any role)", + "description": "lines is capped at 5000.", + "responses": { + "200": { + "description": "OK" + }, + "404": { + "description": "Unknown session" + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/sessions/{id}/input": { + "post": { + "tags": [ + "Sessions" + ], + "summary": "Type text into a session (owner or read-write)", + "description": "Literal paste, never key names. Audited with user and byte count, not the text.", + "responses": { + "200": { + "description": "OK" + }, + "400": { + "description": "Invalid text" + }, + "403": { + "description": "Role not allowed" + }, + "409": { + "description": "Session not running" + }, + "413": { + "description": "Input too large" + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "submit": { + "type": "boolean" + } + } + } + } + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/sessions/{id}/upload": { + "post": { + "tags": [ + "Sessions" + ], + "summary": "Upload a file into a session directory (owner or read-write)", + "description": "Multipart field file. Returns the absolute path; the file is never executed.", + "responses": { + "200": { + "description": "OK" + }, + "403": { + "description": "Role not allowed" + }, + "413": { + "description": "Upload too large" + }, + "429": { + "description": "Per-session file cap reached" + } + }, + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/sessions/{id}/resume": { + "post": { + "tags": [ + "Sessions" + ], + "summary": "Resume an ended session (owner or read-write)", + "description": "Only for vendors with a resume handle.", + "responses": { + "200": { + "description": "OK" + }, + "403": { + "description": "Role not allowed" + }, + "409": { + "description": "Not ended or nothing to resume" + }, + "429": { + "description": "Session limit reached" + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + } } }, "components": { diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 008989016a..f1ec32f26d 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1100,6 +1100,7 @@ func (b *boot) wireBootClosures() { WorkflowClusterRunner: workflowClusterRunner(), WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), AgentMgr: b.agentMgr, + Sessions: cruxruntime.NewUserSessions(b.logger), CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ // linearagent adapters behind the dashboard's consumer-defined diff --git a/src/crux/agent/usersession.go b/src/crux/agent/usersession.go new file mode 100644 index 0000000000..9b4403fdda --- /dev/null +++ b/src/crux/agent/usersession.go @@ -0,0 +1,380 @@ +package agent + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent/resumehandle" + "github.com/hivecommons/hive/crux/agent/tmux" +) + +// UserSessionTmuxPrefix is the tmux session (and socket) name prefix of every +// user session; ttyd-tmux.sh finds the socket by this name. +const UserSessionTmuxPrefix = "hive-sess-" + +const ( + // userSessionCmdTimeout bounds every tmux call so a wedged server cannot + // hold an HTTP handler. + userSessionCmdTimeout = 10 * time.Second + // userSessionBufferName is the tmux paste buffer used for typed input. + userSessionBufferName = "hive-sess-input" + // userSessionMaxCaptureLines caps a scrollback read. + userSessionMaxCaptureLines = fullLogCaptureLines + // userSessionProbeFields is the field count of the probe format below. + userSessionProbeFields = 3 + userSessionProbeFormat = "#{pane_dead} #{session_attached} #{session_activity}" +) + +var ( + userSessionIDPattern = regexp.MustCompile(`^[a-f0-9]{8,32}$`) + // userSessionResumeCmdPattern admits the resume commands the vendors + // print: words, dots, dashes, slashes, colons and equals. + userSessionResumeCmdPattern = regexp.MustCompile(`^[A-Za-z0-9._/:=@-]+( [A-Za-z0-9._/:=@-]+)*$`) +) + +// userSessionVendor describes a CLI a session may run. Args are the vendor's +// plain interactive invocation: NO permission-bypass flag is ever added here, +// which is the point of this table existing separately from +// backends.conf's backend_perm_flag (that function emits the unattended +// bypass flags agents run with, and is deliberately not consulted). +type userSessionVendor struct { + id string + label string + binary string +} + +// userSessionVendors is the interactive-capable subset of KNOWN_BACKENDS +// (config/backends.conf; parity is asserted by a test). Headless runners +// (cline, muse, omnigent), the litellm gateway and any shell are excluded. +var userSessionVendors = []userSessionVendor{ + {"claude", "Claude Code", "claude"}, + {"copilot", "GitHub Copilot CLI", "copilot"}, + {"codex", "Codex", "codex"}, + {"goose", "Goose", "goose"}, + {"agy", "Antigravity", "agy"}, + {"bob", "Bob", "bob"}, + {"pi", "Pi", "pi"}, + {"aider", "Aider", "aider"}, + {"opencode", "OpenCode", "opencode"}, + {"kilo", "Kilo", "kilo"}, + {"omp", "OMP", "omp"}, + {"openhands", "OpenHands", "openhands"}, +} + +func findUserSessionVendor(id string) (userSessionVendor, bool) { + for _, v := range userSessionVendors { + if v.id == id { + return v, true + } + } + return userSessionVendor{}, false +} + +// tmuxRunFunc runs one tmux invocation against a session socket. It is the +// test seam: tests substitute a recorder so no real tmux is ever touched. +type tmuxRunFunc func(ctx context.Context, env []string, stdin, socket string, args ...string) ([]byte, error) + +// UserSessionRuntime starts, probes, types into and kills the tmux sessions +// behind the dashboard's Sessions feature. Each session gets its OWN tmux +// server (socket hive-sess-<id>), its own HOME and a credential-stripped +// environment. It holds no per-session state: the dashboard's store is the +// source of truth for names, groups and ownership. +type UserSessionRuntime struct { + logger *slog.Logger + run tmuxRunFunc + lookPath func(string) (string, error) + environ func() []string + now func() time.Time +} + +// NewUserSessionRuntime returns a runtime that drives the real tmux binary. +func NewUserSessionRuntime(logger *slog.Logger) *UserSessionRuntime { + if logger == nil { + logger = slog.Default() + } + return &UserSessionRuntime{ + logger: logger, + run: execUserSessionTmux, + lookPath: exec.LookPath, + environ: os.Environ, + now: time.Now, + } +} + +func execUserSessionTmux(ctx context.Context, env []string, stdin, socket string, args ...string) ([]byte, error) { + argv := append([]string{"-L", socket}, args...) + cmd := exec.CommandContext(ctx, tmux.Binary, argv...) + cmd.Env = env + cmd.WaitDelay = tmuxCmdWaitDelay + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + out, err := cmd.CombinedOutput() + if err != nil { + return out, outputErr("tmux "+args[0], err, out) + } + return out, nil +} + +// UserSessionTmuxName is the tmux session name for a session id. +func UserSessionTmuxName(id string) string { return UserSessionTmuxPrefix + id } + +func (r *UserSessionRuntime) tmuxEnv() []string { + return userSessionBaseEnv(r.environ(), os.Getenv(userSessionEnvPassthroughEnv)) +} + +func (r *UserSessionRuntime) tmux(id, stdin string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), userSessionCmdTimeout) + defer cancel() + return r.run(ctx, r.tmuxEnv(), stdin, UserSessionTmuxName(id), args...) +} + +func checkUserSessionID(id string) error { + if !userSessionIDPattern.MatchString(id) { + return fmt.Errorf("invalid session id") + } + return nil +} + +// Vendors lists the vendors a session may run with binary presence. +func (r *UserSessionRuntime) Vendors() []crux.UserSessionVendor { + out := make([]crux.UserSessionVendor, 0, len(userSessionVendors)) + for _, v := range userSessionVendors { + _, err := r.lookPath(v.binary) + out = append(out, crux.UserSessionVendor{ID: v.id, Label: v.label, Available: err == nil}) + } + return out +} + +// SupportsResume reports whether the vendor keeps a capturable resume handle. +func (r *UserSessionRuntime) SupportsResume(vendor string) bool { + return BackendSupportsResumeHandle(vendor) +} + +// WorkDir is the session's working directory; the dashboard stores uploads +// beneath it. +func (r *UserSessionRuntime) WorkDir(id string) string { return userSessionWorkDir(id) } + +// Start creates the session's tmux server, strips and sets its environment and +// launches the vendor CLI as the pane's only process (there is no shell to +// fall back to when the CLI exits). +func (r *UserSessionRuntime) Start(spec crux.UserSessionSpec) error { + if err := checkUserSessionID(spec.ID); err != nil { + return err + } + argv, err := r.launchArgv(spec) + if err != nil { + return err + } + // An ended session's dead pane may still hold the tmux server open; a + // resume replaces it. + if p := r.Probe(spec.ID); p.Exists { + if err := r.Kill(spec.ID); err != nil { + return err + } + } + if err := setupUserSessionHome(spec.ID); err != nil { + return err + } + session := UserSessionTmuxName(spec.ID) + create := newSessionCommands(session, userSessionWorkDir(spec.ID)) + // Keep the pane (and its scrollback) after the CLI exits so a late viewer + // can still read what happened; Probe reports it as Dead. + create = append(create, ";", "set-option", "-w", "-t", session, "remain-on-exit", "on") + if out, err := r.tmux(spec.ID, "", create...); err != nil { + return outputErr("creating user session tmux server", err, out) + } + if _, err := r.tmux(spec.ID, "", r.environmentCommands(spec.ID, session)...); err != nil { + _ = r.Kill(spec.ID) + return fmt.Errorf("setting session environment: %w", err) + } + // respawn-pane -k replaces the pre-env shell with the CLI itself, forked + // after the environment was populated. + respawn := append([]string{"respawn-pane", "-k", "-t", session, "-c", userSessionWorkDir(spec.ID), "--"}, argv...) + if _, err := r.tmux(spec.ID, "", respawn...); err != nil { + _ = r.Kill(spec.ID) + return fmt.Errorf("launching %s: %w", spec.Vendor, err) + } + r.logger.Info("user session started", "id", spec.ID, "vendor", spec.Vendor) + return nil +} + +// environmentCommands removes every credential name from the session +// environment and then sets the per-session variables, as one tmux invocation. +func (r *UserSessionRuntime) environmentCommands(id, session string) []string { + var args []string + for _, k := range userSessionStripNames() { + args = append(args, "set-environment", "-t", session, "-r", k, ";") + } + for _, kv := range userSessionEnvPairs(id, r.scrubScript()) { + args = append(args, "set-environment", "-t", session, kv[0], kv[1], ";") + } + return args[:len(args)-1] +} + +func (r *UserSessionRuntime) scrubScript() string { + for _, p := range []string{"/usr/local/bin/agent-env-scrub.sh", filepath.Join(os.Getenv("HIVE_BIN_DIR"), "agent-env-scrub.sh")} { + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p + } + } + return "" +} + +// launchArgv resolves the pane command: the vendor binary alone, or the +// validated resume command for that vendor. Never a shell. +func (r *UserSessionRuntime) launchArgv(spec crux.UserSessionSpec) ([]string, error) { + v, ok := findUserSessionVendor(spec.Vendor) + if !ok { + return nil, fmt.Errorf("unsupported vendor %q", spec.Vendor) + } + bin, err := r.lookPath(v.binary) + if err != nil { + return nil, fmt.Errorf("%s is not installed on this spoke", v.binary) + } + if spec.ResumeCommand == "" { + return []string{bin}, nil + } + if !userSessionResumeCmdPattern.MatchString(spec.ResumeCommand) { + return nil, fmt.Errorf("resume command rejected") + } + fields := strings.Fields(spec.ResumeCommand) + if fields[0] != v.binary { + return nil, fmt.Errorf("resume command does not belong to %s", v.id) + } + return append([]string{bin}, fields[1:]...), nil +} + +// Probe reads the session's live tmux state in one call. +func (r *UserSessionRuntime) Probe(id string) crux.UserSessionProbe { + if checkUserSessionID(id) != nil { + return crux.UserSessionProbe{} + } + out, err := r.tmux(id, "", "display-message", "-p", "-t", UserSessionTmuxName(id), userSessionProbeFormat) + if err != nil { + return crux.UserSessionProbe{} + } + fields := strings.Fields(string(out)) + if len(fields) != userSessionProbeFields { + return crux.UserSessionProbe{Exists: true} + } + attached, _ := strconv.Atoi(fields[1]) + secs, _ := strconv.ParseInt(fields[2], 10, 64) + return crux.UserSessionProbe{ + Exists: true, + Dead: fields[0] == "1", + Attached: attached, + Activity: time.Unix(secs, 0), + } +} + +// Kill ends the session's tmux server. A missing session is not an error. +func (r *UserSessionRuntime) Kill(id string) error { + if err := checkUserSessionID(id); err != nil { + return err + } + if !r.Probe(id).Exists { + return nil + } + if out, err := r.tmux(id, "", "kill-session", "-t", UserSessionTmuxName(id)); err != nil { + return outputErr("killing user session", err, out) + } + return nil +} + +// Capture returns up to lines lines of the session's scrollback as text. +func (r *UserSessionRuntime) Capture(id string, lines int) (string, error) { + if err := checkUserSessionID(id); err != nil { + return "", err + } + if lines <= 0 || lines > userSessionMaxCaptureLines { + lines = userSessionMaxCaptureLines + } + out, err := r.tmux(id, "", "capture-pane", "-t", UserSessionTmuxName(id), "-p", "-J", + "-S", "-"+strconv.Itoa(lines), "-E", "-") + if err != nil { + return "", fmt.Errorf("capturing session: %w", err) + } + return string(out), nil +} + +// SendText types text into the session. It travels through a tmux paste +// buffer (bracketed paste when the CLI asked for it), so nothing is +// interpreted as a key name and a multi-line paste does not execute line by +// line. When submit is true a single Enter follows. The text is sanitised +// first: control and escape characters are refused, because an ESC could close +// the bracketed paste early and turn data into keystrokes. +func (r *UserSessionRuntime) SendText(id, text string, submit bool) error { + if err := checkUserSessionID(id); err != nil { + return err + } + clean, err := SanitizeSessionInput(text) + if err != nil { + return err + } + session := UserSessionTmuxName(id) + if p := r.Probe(id); !p.Exists || p.Dead { + return fmt.Errorf("session is not running") + } + if clean != "" { + if out, err := r.tmux(id, clean, "load-buffer", "-b", userSessionBufferName, "-"); err != nil { + return outputErr("loading input", err, out) + } + if out, err := r.tmux(id, "", "paste-buffer", "-p", "-r", "-d", "-b", userSessionBufferName, "-t", session); err != nil { + return outputErr("pasting input", err, out) + } + } + if submit { + if out, err := r.tmux(id, "", "send-keys", "-t", session, "Enter"); err != nil { + return outputErr("submitting input", err, out) + } + } + return nil +} + +// SanitizeSessionInput validates text for typing into a session: it must be +// valid UTF-8 and contain no control characters other than tab and newline +// (CRLF is folded to LF). It returns the text to paste. +func SanitizeSessionInput(text string) (string, error) { + if !utf8.ValidString(text) { + return "", fmt.Errorf("input is not valid UTF-8") + } + text = strings.ReplaceAll(text, "\r\n", "\n") + var b bytes.Buffer + for _, r := range text { + if r == '\n' || r == '\t' { + b.WriteRune(r) + continue + } + if unicode.IsControl(r) { + return "", fmt.Errorf("input contains a control character") + } + b.WriteRune(r) + } + return b.String(), nil +} + +// ResumeInfo reads the newest vendor-native conversation from the session's +// own HOME. +func (r *UserSessionRuntime) ResumeInfo(id, vendor string) (crux.UserSessionResume, bool) { + if checkUserSessionID(id) != nil || !r.SupportsResume(vendor) { + return crux.UserSessionResume{}, false + } + h, ok := resumehandle.CaptureResumeHandleIn(userSessionHome(id), userSessionCodexHome(id), vendor, userSessionWorkDir(id), r.now()) + if !ok || h.SessionID == "" || h.Command == "" { + return crux.UserSessionResume{}, false + } + return crux.UserSessionResume{SessionID: h.SessionID, Command: h.Command}, true +} diff --git a/src/crux/agent/usersession_env.go b/src/crux/agent/usersession_env.go new file mode 100644 index 0000000000..0a0000ad99 --- /dev/null +++ b/src/crux/agent/usersession_env.go @@ -0,0 +1,139 @@ +package agent + +import ( + "sort" + "strings" +) + +// Session environment policy. A user session is a terminal handed to a person +// through the dashboard, so its environment is built from an ALLOWLIST of the +// hive process env rather than by subtracting known secrets: a credential the +// hive later learns to hold is excluded by default instead of leaking until +// someone remembers to add it to a strip list. +// +// The allowlist covers only what a CLI needs to run and reach the network. +// Vendor API keys are NOT inherited; an operator who wants one in sessions +// names it in HIVE_SESSIONS_ENV_PASSTHROUGH, and even then every name that is +// (or looks like) a hive, dashboard, hub, terminal or GitHub credential is +// refused (sessionEnvForbidden). +const ( + // userSessionEnvPassthroughEnv lists extra env var names (comma separated) + // copied from the hive process into sessions. + userSessionEnvPassthroughEnv = "HIVE_SESSIONS_ENV_PASSTHROUGH" + // userSessionDefaultTerm is the TERM sessions run with; ttyd/xterm.js + // speaks xterm-256color. + userSessionDefaultTerm = "xterm-256color" +) + +// userSessionEnvAllow is the set of hive-process variables copied verbatim. +var userSessionEnvAllow = map[string]bool{ + "PATH": true, + "LANG": true, + "LC_ALL": true, + "LC_CTYPE": true, + "TZ": true, + "TMUX_TMPDIR": true, + "SSL_CERT_FILE": true, + "SSL_CERT_DIR": true, + "NODE_EXTRA_CA_CERTS": true, + "REQUESTS_CA_BUNDLE": true, + "CURL_CA_BUNDLE": true, +} + +// userSessionProxyVars are copied only when their value carries no userinfo +// (a proxy URL's user:password is a credential). +var userSessionProxyVars = map[string]bool{ + "HTTP_PROXY": true, "HTTPS_PROXY": true, "NO_PROXY": true, "ALL_PROXY": true, + "http_proxy": true, "https_proxy": true, "no_proxy": true, "all_proxy": true, +} + +// sessionEnvForbiddenNames are refused even through the operator passthrough. +func sessionEnvForbiddenNames() map[string]bool { + out := map[string]bool{ + "DASHBOARD_AUTH_TOKEN": true, + "HIVE_DASHBOARD_TOKEN": true, + } + for _, k := range inheritedCredentialEnvVars { + out[k] = true + } + for _, k := range githubTokenEnvVars { + out[k] = true + } + return out +} + +// sessionEnvForbidden reports whether name must never reach a session: a +// known hive credential, anything in the HIVE_ namespace (hub, heartbeat, +// terminal and ttyd secrets live there), or a GitHub token spelling. +func sessionEnvForbidden(name string) bool { + if sessionEnvForbiddenNames()[name] { + return true + } + upper := strings.ToUpper(name) + return strings.HasPrefix(upper, "HIVE_") || + strings.HasPrefix(upper, "GITHUB_") || + strings.HasPrefix(upper, "GH_") || + strings.Contains(upper, "GITHUB_TOKEN") +} + +// userSessionStripNames is the sorted list handed to `tmux set-environment -r` +// as a second line of defence: even if the tmux server somehow inherited one, +// the removal hides it from every process the session forks. +func userSessionStripNames() []string { + names := sessionEnvForbiddenNames() + out := make([]string, 0, len(names)) + for k := range names { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// userSessionBaseEnv builds the environment a session's tmux server starts +// with from the hive process environ (KEY=VALUE entries) and the operator's +// passthrough list. +func userSessionBaseEnv(environ []string, passthrough string) []string { + extra := map[string]bool{} + for _, n := range strings.Split(passthrough, ",") { + n = strings.TrimSpace(n) + if n != "" && !sessionEnvForbidden(n) { + extra[n] = true + } + } + var out []string + for _, kv := range environ { + k, v, ok := strings.Cut(kv, "=") + if !ok || k == "" { + continue + } + switch { + case sessionEnvForbidden(k): + continue + case userSessionEnvAllow[k], extra[k]: + out = append(out, k+"="+v) + case userSessionProxyVars[k] && proxyValueHasNoUserinfo(v): + out = append(out, k+"="+v) + } + } + out = append(out, "TERM="+userSessionDefaultTerm) + return out +} + +// proxyValueHasNoUserinfo reports whether a proxy setting is safe to copy. +// NO_PROXY style lists have no "@" and pass. Anything else is judged on its +// authority component (after an optional scheme, up to the first path, query +// or fragment delimiter) so a schemeless "user:pw@host:port" is caught too; +// url.Parse would read "user" as a scheme and miss the credentials. +func proxyValueHasNoUserinfo(v string) bool { + if !strings.Contains(v, "@") { + return true + } + rest := v + if i := strings.Index(rest, "://"); i >= 0 { + rest = rest[i+len("://"):] + } + if end := strings.IndexAny(rest, "/?#"); end >= 0 { + rest = rest[:end] + } + return !strings.Contains(rest, "@") +} diff --git a/src/crux/agent/usersession_home.go b/src/crux/agent/usersession_home.go new file mode 100644 index 0000000000..d0c8d10e8b --- /dev/null +++ b/src/crux/agent/usersession_home.go @@ -0,0 +1,116 @@ +package agent + +import ( + "fmt" + "os" + "path/filepath" +) + +// Per-session filesystem layout. A session never shares a HOME with another +// session or with an agent, so a vendor CLI's transcripts, caches and memory +// stay per-session. Vendor LOGINS are shared fleet-wide by bridging only the +// credential files a vendor CLI needs (the perAgentCLIHomes bridge lists the +// fleet already trusts) from the shared home, never the whole shared +// ~/.config (which holds gh's hosts.yml). +const ( + userSessionHomeDirName = "sessions" + userSessionHomeMode = 0o700 + userSessionWorkDirName = "work" + // userSessionsDirEnv relocates the sessions data root (default + // /data/sessions), shared with the dashboard store. + userSessionsDirEnv = "HIVE_SESSIONS_DIR" + defaultUserSessionsDir = "/data/sessions" +) + +// sharedConfigBridges are the ~/.config subtrees bridged from the shared home +// because a vendor stores its provider login there. gh is deliberately absent. +var sharedConfigBridges = []string{"goose", "opencode", "aider"} + +// userSessionsDir is the data root holding sessions.json and per-session work +// dirs. +func userSessionsDir() string { + if v := os.Getenv(userSessionsDirEnv); v != "" { + return v + } + return defaultUserSessionsDir +} + +func userSessionHome(id string) string { + return filepath.Join(sharedAgentHome, userSessionHomeDirName, id) +} + +func userSessionCodexHome(id string) string { + return filepath.Join(userSessionHome(id), ".codex") +} + +func userSessionWorkDir(id string) string { + return filepath.Join(userSessionsDir(), id, userSessionWorkDirName) +} + +// setupUserSessionHome creates the per-session HOME, XDG dirs, CODEX_HOME and +// work dir and bridges the shared vendor login files into it. +func setupUserSessionHome(id string) error { + home := userSessionHome(id) + dirs := []string{ + home, + filepath.Join(home, ".config"), + agentXDGDataHome(home), + agentXDGStateHome(home), + userSessionCodexHome(id), + userSessionWorkDir(id), + } + for _, d := range dirs { + if err := os.MkdirAll(d, userSessionHomeMode); err != nil { + return fmt.Errorf("creating session dir %s: %w", d, err) + } + } + for _, spec := range perAgentCLIHomes { + dir := filepath.Join(home, spec.dir) + if err := os.MkdirAll(dir, userSessionHomeMode); err != nil { + return fmt.Errorf("creating %s: %w", dir, err) + } + for _, name := range spec.bridges { + bridgeSessionEntry(filepath.Join(dir, name), filepath.Join(sharedAgentHome, spec.dir, name)) + } + } + for _, name := range sharedConfigBridges { + bridgeSessionEntry(filepath.Join(home, ".config", name), filepath.Join(sharedAgentHome, ".config", name)) + } + // codex keeps its login in CODEX_HOME/auth.json. + bridgeSessionEntry(filepath.Join(userSessionCodexHome(id), "auth.json"), filepath.Join(sharedAgentHome, ".codex", "auth.json")) + return nil +} + +// bridgeSessionEntry links link -> target when the shared target exists and +// link is absent. A missing target is skipped (the vendor was never signed in +// fleet-wide); an existing link is left alone so a re-launch keeps whatever the +// CLI wrote. +func bridgeSessionEntry(link, target string) { + if _, err := os.Stat(target); err != nil { + return + } + if _, err := os.Lstat(link); err == nil { + return + } + _ = os.Symlink(target, link) +} + +// userSessionEnvPairs are the per-session variables set on the tmux session +// after the inherited credentials were stripped. +func userSessionEnvPairs(id, scrubScript string) [][2]string { + home := userSessionHome(id) + pairs := [][2]string{ + {"HOME", home}, + {"CODEX_HOME", userSessionCodexHome(id)}, + {"XDG_CONFIG_HOME", filepath.Join(home, ".config")}, + {"XDG_DATA_HOME", agentXDGDataHome(home)}, + {"XDG_STATE_HOME", agentXDGStateHome(home)}, + {"HIVE_SESSION_ID", id}, + } + if scrubScript != "" { + // Sourced by every non-interactive bash the CLI spawns; unsets any + // GitHub token the CLI re-exports into its tool shells (#4045). + pairs = append(pairs, [2]string{"BASH_ENV", scrubScript}, [2]string{"ENV", scrubScript}) + } + return pairs +} diff --git a/src/crux/agent/usersession_more_test.go b/src/crux/agent/usersession_more_test.go new file mode 100644 index 0000000000..611361e3c5 --- /dev/null +++ b/src/crux/agent/usersession_more_test.go @@ -0,0 +1,509 @@ +package agent + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" +) + +const ( + testResumeUUID = "123e4567-e89b-12d3-a456-426614174000" + testFakeTmuxScript = "#!/bin/sh\necho \"args=$*\"\ncat\ncase \"$3\" in fail) echo boom; exit 3;; esac\n" + testScrubScript = "agent-env-scrub.sh" +) + +// sessionLifecycleTmux fakes a tmux server that only "exists" once new-session +// ran, and fails any call whose argv contains failOn. +func sessionLifecycleTmux(failOn string) *fakeTmux { + created := false + f := &fakeTmux{} + f.handler = func(args []string) ([]byte, error) { + joined := strings.Join(args, " ") + if failOn != "" && strings.Contains(joined, failOn) { + return []byte("simulated failure"), errors.New("tmux failed") + } + switch { + case args[0] == "display-message": + if !created { + return nil, errors.New("no session") + } + return []byte("0 0 1\n"), nil + case strings.Contains(joined, "new-session"): + created = true + } + return nil, nil + } + return f +} + +func TestUserSessionStartStepFailuresAreReportedAndCleanedUp(t *testing.T) { + cases := []struct { + name, failOn, wantErr string + wantKill bool + }{ + {"create", "remain-on-exit", "creating user session tmux server", false}, + {"environment", "set-environment", "setting session environment", true}, + {"respawn", "respawn-pane", "launching claude", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := sessionLifecycleTmux(tc.failOn) + r := newTestSessionRuntime(t, f, nil) + err := r.Start(crux.UserSessionSpec{ID: testSessionID, Vendor: "claude"}) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Start error = %v, want it to mention %q", err, tc.wantErr) + } + killed := false + for _, v := range f.verbs() { + if v == "kill-session" { + killed = true + } + } + if killed != tc.wantKill { + t.Errorf("kill-session issued = %v, want %v (calls %v)", killed, tc.wantKill, f.verbs()) + } + }) + } +} + +func TestUserSessionStartReplacesExistingSession(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { + if args[0] == "display-message" { + return []byte("1 0 1\n"), nil + } + return nil, nil + }} + r := newTestSessionRuntime(t, f, nil) + if err := r.Start(crux.UserSessionSpec{ID: testSessionID, Vendor: "claude"}); err != nil { + t.Fatal(err) + } + verbs := f.verbs() + killAt, respawnAt := -1, -1 + for i, v := range verbs { + switch v { + case "kill-session": + killAt = i + case "respawn-pane": + respawnAt = i + } + } + if killAt < 0 || respawnAt < 0 || killAt > respawnAt { + t.Errorf("dead session was not killed before relaunch: %v", verbs) + } +} + +func TestUserSessionStartSurfacesKillFailure(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { + switch args[0] { + case "display-message": + return []byte("1 0 1\n"), nil + case "kill-session": + return []byte("server wedged"), errors.New("exit 1") + } + return nil, nil + }} + r := newTestSessionRuntime(t, f, nil) + err := r.Start(crux.UserSessionSpec{ID: testSessionID, Vendor: "claude"}) + if err == nil || !strings.Contains(err.Error(), "killing user session") || !strings.Contains(err.Error(), "server wedged") { + t.Fatalf("Start error = %v, want kill failure with tmux output", err) + } + for _, v := range f.verbs() { + if v == "respawn-pane" { + t.Error("CLI launched despite the old session not being killed") + } + } +} + +func TestUserSessionStartFailsWhenHomeCannotBeCreated(t *testing.T) { + f := sessionLifecycleTmux("") + r := newTestSessionRuntime(t, f, nil) + blocker := filepath.Join(t.TempDir(), "blocker") + if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + sharedAgentHome = filepath.Join(blocker, "home") + err := r.Start(crux.UserSessionSpec{ID: testSessionID, Vendor: "claude"}) + if err == nil || !strings.Contains(err.Error(), "creating session dir") { + t.Fatalf("Start error = %v, want session dir failure", err) + } + for _, v := range f.verbs() { + if v == "respawn-pane" || v == "set-option" { + t.Errorf("tmux %s ran after home setup failed", v) + } + } +} + +func TestUserSessionStartRequiresInstalledBinary(t *testing.T) { + f := &fakeTmux{} + r := newTestSessionRuntime(t, f, nil) + r.lookPath = func(string) (string, error) { return "", errors.New("not found") } + err := r.Start(crux.UserSessionSpec{ID: testSessionID, Vendor: "claude"}) + if err == nil || !strings.Contains(err.Error(), "claude is not installed") { + t.Fatalf("Start error = %v", err) + } + if len(f.calls) != 0 { + t.Errorf("tmux reached for a missing binary: %v", f.verbs()) + } +} + +func TestUserSessionLaunchArgv(t *testing.T) { + r := newTestSessionRuntime(t, &fakeTmux{}, nil) + cases := []struct { + name string + spec crux.UserSessionSpec + want []string + wantErr string + }{ + {"bare", crux.UserSessionSpec{Vendor: "codex"}, []string{"/usr/bin/codex"}, ""}, + {"resume", crux.UserSessionSpec{Vendor: "codex", ResumeCommand: "codex resume abc-1"}, []string{"/usr/bin/codex", "resume", "abc-1"}, ""}, + {"unknown vendor", crux.UserSessionSpec{Vendor: "nope"}, nil, `unsupported vendor "nope"`}, + {"metacharacters", crux.UserSessionSpec{Vendor: "claude", ResumeCommand: "claude --resume a|b"}, nil, "resume command rejected"}, + {"leading space", crux.UserSessionSpec{Vendor: "claude", ResumeCommand: " claude --resume a"}, nil, "resume command rejected"}, + {"wrong binary", crux.UserSessionSpec{Vendor: "claude", ResumeCommand: "copilot --resume a"}, nil, "does not belong to claude"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := r.launchArgv(tc.spec) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want %q", err, tc.wantErr) + } + return + } + if err != nil || strings.Join(got, " ") != strings.Join(tc.want, " ") { + t.Fatalf("argv = %v, %v; want %v", got, err, tc.want) + } + }) + } +} + +func TestUserSessionEnvironmentCommandsStripThenSet(t *testing.T) { + r := newTestSessionRuntime(t, &fakeTmux{}, nil) + session := UserSessionTmuxName(testSessionID) + args := r.environmentCommands(testSessionID, session) + if len(args) == 0 || args[len(args)-1] == ";" { + t.Fatalf("command list must not end with a separator: tail %v", args[len(args)-2:]) + } + joined := strings.Join(args, " ") + lastStrip := strings.LastIndex(joined, "-r ") + firstSet := strings.Index(joined, "set-environment -t "+session+" HOME ") + if firstSet < 0 || lastStrip > firstSet { + t.Errorf("credentials must be removed before per-session vars are set:\n%s", joined) + } + for _, name := range userSessionStripNames() { + if !strings.Contains(joined, "-r "+name+" ;") { + t.Errorf("strip of %s missing", name) + } + } + if !strings.Contains(joined, "HIVE_SESSION_ID "+testSessionID) { + t.Error("HIVE_SESSION_ID not set") + } +} + +func TestUserSessionScrubScriptDiscovery(t *testing.T) { + if _, err := os.Stat("/usr/local/bin/" + testScrubScript); err == nil { + t.Skip("host has a system-wide scrub script") + } + r := newTestSessionRuntime(t, &fakeTmux{}, nil) + binDir := t.TempDir() + t.Setenv("HIVE_BIN_DIR", binDir) + if got := r.scrubScript(); got != "" { + t.Fatalf("scrubScript() = %q with no script present", got) + } + // A directory of that name is not a script. + if err := os.Mkdir(filepath.Join(binDir, testScrubScript), 0o755); err != nil { + t.Fatal(err) + } + if got := r.scrubScript(); got != "" { + t.Fatalf("scrubScript() accepted a directory: %q", got) + } + if err := os.Remove(filepath.Join(binDir, testScrubScript)); err != nil { + t.Fatal(err) + } + script := filepath.Join(binDir, testScrubScript) + if err := os.WriteFile(script, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + if got := r.scrubScript(); got != script { + t.Fatalf("scrubScript() = %q, want %q", got, script) + } + joined := strings.Join(r.environmentCommands(testSessionID, UserSessionTmuxName(testSessionID)), " ") + if !strings.Contains(joined, "BASH_ENV "+script) || !strings.Contains(joined, "ENV "+script) { + t.Errorf("scrub script not wired into BASH_ENV/ENV: %s", joined) + } +} + +func TestUserSessionEnvPairs(t *testing.T) { + newTestSessionRuntime(t, &fakeTmux{}, nil) + toMap := func(pairs [][2]string) map[string]string { + m := map[string]string{} + for _, p := range pairs { + m[p[0]] = p[1] + } + return m + } + plain := toMap(userSessionEnvPairs(testSessionID, "")) + if plain["HOME"] != userSessionHome(testSessionID) || plain["CODEX_HOME"] != userSessionCodexHome(testSessionID) || + plain["XDG_CONFIG_HOME"] != filepath.Join(userSessionHome(testSessionID), ".config") || + plain["HIVE_SESSION_ID"] != testSessionID { + t.Errorf("pairs = %v", plain) + } + if _, ok := plain["BASH_ENV"]; ok { + t.Error("BASH_ENV set without a scrub script") + } + scrubbed := toMap(userSessionEnvPairs(testSessionID, "/opt/scrub.sh")) + if scrubbed["BASH_ENV"] != "/opt/scrub.sh" || scrubbed["ENV"] != "/opt/scrub.sh" { + t.Errorf("scrub pairs = %v", scrubbed) + } +} + +func TestUserSessionPathHelpers(t *testing.T) { + newTestSessionRuntime(t, &fakeTmux{}, nil) + t.Setenv(userSessionsDirEnv, "") + if got := userSessionsDir(); got != defaultUserSessionsDir { + t.Errorf("default sessions dir = %q", got) + } + root := t.TempDir() + t.Setenv(userSessionsDirEnv, root) + if got := userSessionsDir(); got != root { + t.Errorf("sessions dir override = %q", got) + } + r := &UserSessionRuntime{} + want := filepath.Join(root, testSessionID, userSessionWorkDirName) + if got := r.WorkDir(testSessionID); got != want { + t.Errorf("WorkDir = %q, want %q", got, want) + } + if got := userSessionCodexHome(testSessionID); got != filepath.Join(userSessionHome(testSessionID), ".codex") { + t.Errorf("codex home = %q", got) + } + if !strings.HasPrefix(userSessionHome(testSessionID), sharedAgentHome) { + t.Errorf("session home %q outside shared home", userSessionHome(testSessionID)) + } +} + +func TestCheckUserSessionID(t *testing.T) { + for _, id := range []string{testSessionID, strings.Repeat("a", 8), strings.Repeat("f", 32)} { + if err := checkUserSessionID(id); err != nil { + t.Errorf("checkUserSessionID(%q) = %v", id, err) + } + } + for _, id := range []string{"", "abc", strings.Repeat("a", 33), "ABCDEF0123", "0123456789ab/..", "0123456789zz", "0123456789ab\n"} { + if err := checkUserSessionID(id); err == nil { + t.Errorf("checkUserSessionID(%q) accepted", id) + } + } +} + +func TestUserSessionSupportsResume(t *testing.T) { + r := &UserSessionRuntime{} + if !r.SupportsResume("claude") || !r.SupportsResume("codex") { + t.Error("claude and codex keep resume handles") + } + if r.SupportsResume("") || r.SupportsResume("bash") { + t.Error("unknown vendors must not report resume support") + } +} + +func TestUserSessionResumeInfo(t *testing.T) { + r := newTestSessionRuntime(t, &fakeTmux{}, nil) + if _, ok := r.ResumeInfo("../x", "claude"); ok { + t.Error("invalid id produced resume info") + } + if _, ok := r.ResumeInfo(testSessionID, "bash"); ok { + t.Error("unsupported vendor produced resume info") + } + if _, ok := r.ResumeInfo(testSessionID, "claude"); ok { + t.Error("session without transcripts produced resume info") + } + dir := filepath.Join(userSessionHome(testSessionID), ".claude", "projects", "proj") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, testResumeUUID+".jsonl"), []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + got, ok := r.ResumeInfo(testSessionID, "claude") + if !ok || got.SessionID != testResumeUUID || got.Command != "claude --resume "+testResumeUUID { + t.Fatalf("ResumeInfo = %+v, %v", got, ok) + } + // Another session's transcripts are invisible: homes are per session. + if _, ok := r.ResumeInfo("fedcba987654", "claude"); ok { + t.Error("resume info leaked across session homes") + } +} + +func TestUserSessionBaseEnvProxyAndMalformedEntries(t *testing.T) { + env := userSessionBaseEnv([]string{ + "PATH=/usr/bin", + "NOEQUALS", + "=emptykey", + "https_proxy=http://proxy.example:3128", + "HTTP_PROXY=http://user:pw@proxy.example:3128", + "NO_PROXY=localhost,10.0.0.0/8", + "CUSTOM_VENDOR_KEY=abc", + "LANG=C.UTF-8", + }, " ,CUSTOM_VENDOR_KEY, ") + joined := strings.Join(env, "\n") + for _, want := range []string{"PATH=/usr/bin", "https_proxy=http://proxy.example:3128", "NO_PROXY=localhost,10.0.0.0/8", "CUSTOM_VENDOR_KEY=abc", "LANG=C.UTF-8", "TERM=" + userSessionDefaultTerm} { + if !strings.Contains(joined, want) { + t.Errorf("env lacks %q:\n%s", want, joined) + } + } + for _, bad := range []string{"NOEQUALS", "emptykey", "pw@"} { + if strings.Contains(joined, bad) { + t.Errorf("env contains %q:\n%s", bad, joined) + } + } + if env[len(env)-1] != "TERM="+userSessionDefaultTerm { + t.Errorf("TERM must be last, got %v", env) + } +} + +func TestProxyValueHasNoUserinfo(t *testing.T) { + cases := map[string]bool{ + "": true, + "localhost,10.0.0.0/8": true, + "http://proxy.example:3128": true, + "http://user@proxy.example": false, + "http://user:pw@proxy.example": false, + "user:pw@proxy.example:3128": false, + "http://u@[::1": false, + "http://proxy.example/path?a@b": true, + } + for in, want := range cases { + if got := proxyValueHasNoUserinfo(in); got != want { + t.Errorf("proxyValueHasNoUserinfo(%q) = %v, want %v", in, got, want) + } + } +} + +func TestBridgeSessionEntry(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "shared") + link := filepath.Join(dir, "link") + + bridgeSessionEntry(link, target) + if _, err := os.Lstat(link); err == nil { + t.Fatal("link created for a missing target") + } + if err := os.WriteFile(target, []byte("login"), 0o600); err != nil { + t.Fatal(err) + } + bridgeSessionEntry(link, target) + if got, err := os.Readlink(link); err != nil || got != target { + t.Fatalf("Readlink = %q, %v", got, err) + } + // An existing entry is left alone, even when it points elsewhere. + other := filepath.Join(dir, "other") + if err := os.WriteFile(other, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Remove(link); err != nil { + t.Fatal(err) + } + if err := os.Symlink(other, link); err != nil { + t.Fatal(err) + } + bridgeSessionEntry(link, target) + if got, _ := os.Readlink(link); got != other { + t.Errorf("existing link overwritten: %q", got) + } +} + +func TestSetupUserSessionHomeLayoutAndIdempotence(t *testing.T) { + newTestSessionRuntime(t, &fakeTmux{}, nil) + shared := filepath.Join(sharedAgentHome, ".config", "goose") + if err := os.MkdirAll(shared, 0o755); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := setupUserSessionHome(testSessionID); err != nil { + t.Fatalf("setup #%d: %v", i, err) + } + } + home := userSessionHome(testSessionID) + for _, d := range []string{home, filepath.Join(home, ".config"), agentXDGDataHome(home), agentXDGStateHome(home), userSessionCodexHome(testSessionID), userSessionWorkDir(testSessionID)} { + if st, err := os.Stat(d); err != nil || !st.IsDir() { + t.Errorf("%s not created: %v", d, err) + } + } + if got, err := os.Readlink(filepath.Join(home, ".config", "goose")); err != nil || got != shared { + t.Errorf("goose bridge = %q, %v", got, err) + } + if _, err := os.Lstat(filepath.Join(home, ".config", "opencode")); err == nil { + t.Error("bridge created for a vendor that was never signed in") + } +} + +func TestSetupUserSessionHomeFailsWhenVendorDirBlocked(t *testing.T) { + newTestSessionRuntime(t, &fakeTmux{}, nil) + home := userSessionHome(testSessionID) + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatal(err) + } + blocked := filepath.Join(home, perAgentCLIHomes[0].dir) + if err := os.WriteFile(blocked, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + err := setupUserSessionHome(testSessionID) + if err == nil || !strings.Contains(err.Error(), blocked) { + t.Fatalf("setup error = %v, want failure naming %s", err, blocked) + } +} + +func TestNewUserSessionRuntimeWiresRealDependencies(t *testing.T) { + r := NewUserSessionRuntime(nil) + if r.logger == nil || r.run == nil || r.lookPath == nil || r.environ == nil || r.now == nil { + t.Fatalf("runtime not fully wired: %+v", r) + } + logger := slog.Default() + if got := NewUserSessionRuntime(logger); got.logger != logger { + t.Error("supplied logger not used") + } + t.Setenv("HIVE_TEST_SESSION_ENVIRON_MARKER", "1") + found := false + for _, kv := range r.environ() { + if kv == "HIVE_TEST_SESSION_ENVIRON_MARKER=1" { + found = true + } + } + if !found { + t.Error("environ is not the process environment") + } +} + +func TestExecUserSessionTmux(t *testing.T) { + binDir := t.TempDir() + if err := os.WriteFile(filepath.Join(binDir, "tmux"), []byte(testFakeTmuxScript), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+":/bin:/usr/bin") + env := []string{"PATH=/bin:/usr/bin"} + + out, err := execUserSessionTmux(context.Background(), env, "typed text", "hive-sess-x", "load-buffer", "-") + if err != nil { + t.Fatalf("err = %v, out %q", err, out) + } + if !strings.Contains(string(out), "args=-L hive-sess-x load-buffer -") || !strings.Contains(string(out), "typed text") { + t.Errorf("socket flag or stdin not passed: %q", out) + } + + out, err = execUserSessionTmux(context.Background(), env, "", "hive-sess-x", "fail") + if err == nil || !strings.Contains(err.Error(), "tmux fail") || !strings.Contains(err.Error(), "boom") { + t.Fatalf("err = %v, want wrapped failure with tmux output", err) + } + if !strings.Contains(string(out), "boom") { + t.Errorf("output not returned on failure: %q", out) + } + + t.Setenv("PATH", t.TempDir()) + if _, err := execUserSessionTmux(context.Background(), env, "", "hive-sess-x", "display-message"); err == nil || !strings.Contains(err.Error(), "tmux display-message") { + t.Errorf("missing binary error = %v", err) + } +} diff --git a/src/crux/agent/usersession_test.go b/src/crux/agent/usersession_test.go new file mode 100644 index 0000000000..3d282a7c44 --- /dev/null +++ b/src/crux/agent/usersession_test.go @@ -0,0 +1,474 @@ +package agent + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" +) + +const testSessionID = "0123456789ab" + +type fakeTmuxCall struct { + env []string + stdin string + socket string + args []string +} + +// fakeTmux is a deterministic stand-in for the tmux binary. handler may be +// nil (every call succeeds with empty output). +type fakeTmux struct { + calls []fakeTmuxCall + handler func(args []string) ([]byte, error) +} + +func (f *fakeTmux) run(_ context.Context, env []string, stdin, socket string, args ...string) ([]byte, error) { + f.calls = append(f.calls, fakeTmuxCall{env: env, stdin: stdin, socket: socket, args: args}) + if f.handler != nil { + return f.handler(args) + } + return nil, nil +} + +func (f *fakeTmux) verbs() []string { + var out []string + for _, c := range f.calls { + out = append(out, c.args[0]) + } + return out +} + +func newTestSessionRuntime(t *testing.T, f *fakeTmux, environ []string) *UserSessionRuntime { + t.Helper() + oldHome := sharedAgentHome + sharedAgentHome = filepath.Join(t.TempDir(), "home") + t.Cleanup(func() { sharedAgentHome = oldHome }) + t.Setenv(userSessionsDirEnv, filepath.Join(t.TempDir(), "sessions")) + t.Setenv(userSessionEnvPassthroughEnv, "") + return &UserSessionRuntime{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + run: f.run, + lookPath: func(name string) (string, error) { return "/usr/bin/" + name, nil }, + environ: func() []string { return environ }, + now: time.Now, + } +} + +var sessionSecretEnv = []string{ + "PATH=/usr/bin", + "DASHBOARD_AUTH_TOKEN=dash-secret", + "HIVE_DASHBOARD_TOKEN=dash-secret-2", + "HIVE_TERMINAL_KEY=term-secret", + "HIVE_HUB_SECRET=hub-secret", + "HIVE_HEARTBEAT_KEY=hb-secret", + "HIVE_TTYD_CREDENTIAL=ttyd-secret", + "HIVE_INVITE_KEY=invite-secret", + "HIVE_GITHUB_TOKEN=ghs_installtoken", + "GH_TOKEN=ghp_pat", + "GITHUB_TOKEN=ghs_other", + "GH_ENTERPRISE_TOKEN=ghe", + "ANTHROPIC_API_KEY=sk-ant-secret", + "HTTPS_PROXY=http://user:proxypass@proxy.example:3128", +} + +// TestUserSessionBaseEnvStripsEveryCredential asserts the invariant itself: +// whatever the hive process env holds, no session env entry carries a +// dashboard, terminal, hub, heartbeat, ttyd or GitHub credential, by name or by +// value. +func TestUserSessionBaseEnvStripsEveryCredential(t *testing.T) { + env := userSessionBaseEnv(sessionSecretEnv, "") + joined := strings.Join(env, "\n") + for _, secret := range []string{ + "dash-secret", "term-secret", "hub-secret", "hb-secret", "ttyd-secret", "invite-secret", + "ghs_installtoken", "ghp_pat", "ghs_other", "sk-ant-secret", "proxypass", + } { + if strings.Contains(joined, secret) { + t.Errorf("session env leaks %q:\n%s", secret, joined) + } + } + if !strings.Contains(joined, "PATH=/usr/bin") { + t.Errorf("session env lost PATH: %v", env) + } + if !strings.Contains(joined, "TERM="+userSessionDefaultTerm) { + t.Errorf("session env missing TERM: %v", env) + } +} + +func TestUserSessionBaseEnvPassthroughCannotReintroduceCredentials(t *testing.T) { + env := userSessionBaseEnv(sessionSecretEnv, "ANTHROPIC_API_KEY, HIVE_TERMINAL_KEY, GH_TOKEN, DASHBOARD_AUTH_TOKEN, HIVE_FUTURE_SECRET") + joined := strings.Join(env, "\n") + if !strings.Contains(joined, "ANTHROPIC_API_KEY=sk-ant-secret") { + t.Errorf("operator passthrough of a vendor key was dropped: %v", env) + } + for _, secret := range []string{"term-secret", "ghp_pat", "dash-secret"} { + if strings.Contains(joined, secret) { + t.Errorf("passthrough leaked %q", secret) + } + } +} + +func TestUserSessionForbiddenCoversEveryKnownCredentialVar(t *testing.T) { + strip := map[string]bool{} + for _, n := range userSessionStripNames() { + strip[n] = true + } + var all []string + all = append(all, inheritedCredentialEnvVars...) + all = append(all, githubTokenEnvVars...) + all = append(all, "DASHBOARD_AUTH_TOKEN", "HIVE_DASHBOARD_TOKEN", "HIVE_TERMINAL_KEY", "HIVE_HUB_SECRET", "HIVE_HEARTBEAT_KEY", "HIVE_TTYD_CREDENTIAL") + for _, name := range all { + if !sessionEnvForbidden(name) { + t.Errorf("%s is not forbidden in sessions", name) + } + if !strip[name] { + t.Errorf("%s is not in the set-environment -r strip list", name) + } + } +} + +func TestUserSessionStartLaunchesBareCLIWithoutBypassFlagsOrShell(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { + if args[0] == "display-message" { + return []byte("can't find session"), errors.New("no session") + } + return nil, nil + }} + r := newTestSessionRuntime(t, f, sessionSecretEnv) + for _, v := range userSessionVendors { + f.calls = nil + if err := r.Start(crux.UserSessionSpec{ID: testSessionID, Vendor: v.id}); err != nil { + t.Fatalf("Start(%s): %v", v.id, err) + } + var respawn []string + var envCmd []string + for _, c := range f.calls { + if c.socket != UserSessionTmuxPrefix+testSessionID { + t.Fatalf("socket = %q, want per-session socket", c.socket) + } + joined := strings.Join(c.env, "\n") + for _, secret := range []string{"dash-secret", "term-secret", "hub-secret", "ghp_pat", "ghs_installtoken", "proxypass"} { + if strings.Contains(joined, secret) { + t.Fatalf("tmux client env leaks %q", secret) + } + } + if c.args[0] == "respawn-pane" { + respawn = c.args + } + for _, a := range c.args { + if a == "set-environment" { + envCmd = c.args + } + } + } + if respawn == nil { + t.Fatalf("%s: no respawn-pane call in %v", v.id, f.verbs()) + } + idx := -1 + for i, a := range respawn { + if a == "--" { + idx = i + } + } + if idx < 0 { + t.Fatalf("%s: respawn has no -- separator: %v", v.id, respawn) + } + argv := respawn[idx+1:] + if len(argv) != 1 || argv[0] != "/usr/bin/"+v.binary { + t.Errorf("%s: pane command = %v, want only the bare binary", v.id, argv) + } + for _, a := range respawn { + if strings.Contains(a, "allow-all") || strings.Contains(a, "bypass") || strings.Contains(a, "yolo") || + strings.Contains(a, "dangerously") || strings.Contains(a, "auto-approve") { + t.Errorf("%s: launch carries a permission-bypass flag: %v", v.id, respawn) + } + if a == "bash" || a == "sh" || a == "zsh" || strings.HasSuffix(a, "/bash") || strings.HasSuffix(a, "/sh") { + t.Errorf("%s: launch reaches a shell: %v", v.id, respawn) + } + } + if envCmd == nil { + t.Fatalf("%s: no set-environment call", v.id) + } + joined := strings.Join(envCmd, " ") + for _, name := range []string{"HIVE_TERMINAL_KEY", "DASHBOARD_AUTH_TOKEN", "GH_TOKEN", "HIVE_HUB_SECRET"} { + if !strings.Contains(joined, "-r "+name) { + t.Errorf("%s: session env does not remove %s", v.id, name) + } + } + if !strings.Contains(joined, "HOME "+userSessionHome(testSessionID)) { + t.Errorf("%s: session HOME not set to the per-session dir", v.id) + } + } + if _, err := os.Stat(userSessionWorkDir(testSessionID)); err != nil { + t.Errorf("work dir not created: %v", err) + } +} + +func TestUserSessionStartRejectsBadInput(t *testing.T) { + f := &fakeTmux{} + r := newTestSessionRuntime(t, f, nil) + cases := []crux.UserSessionSpec{ + {ID: "../etc", Vendor: "claude"}, + {ID: "XYZ", Vendor: "claude"}, + {ID: testSessionID, Vendor: "bash"}, + {ID: testSessionID, Vendor: "sh"}, + {ID: testSessionID, Vendor: "claude", ResumeCommand: "claude --resume abc; rm -rf /"}, + {ID: testSessionID, Vendor: "claude", ResumeCommand: "codex resume abc"}, + {ID: testSessionID, Vendor: "claude", ResumeCommand: "claude --resume $(id)"}, + } + for _, spec := range cases { + if err := r.Start(spec); err == nil { + t.Errorf("Start(%+v) succeeded, want error", spec) + } + } + if len(f.calls) != 0 { + t.Errorf("rejected starts reached tmux: %v", f.verbs()) + } +} + +func TestUserSessionStartResumeUsesValidatedCommand(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { + if args[0] == "display-message" { + return nil, errors.New("no session") + } + return nil, nil + }} + r := newTestSessionRuntime(t, f, nil) + spec := crux.UserSessionSpec{ID: testSessionID, Vendor: "claude", ResumeCommand: "claude --resume 123e4567-e89b-12d3-a456-426614174000"} + if err := r.Start(spec); err != nil { + t.Fatal(err) + } + last := f.calls[len(f.calls)-1].args + joined := strings.Join(last, " ") + if !strings.HasSuffix(joined, "-- /usr/bin/claude --resume 123e4567-e89b-12d3-a456-426614174000") { + t.Errorf("resume launch = %q", joined) + } +} + +func TestUserSessionProbe(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { + return []byte("0 2 1700000000\n"), nil + }} + r := newTestSessionRuntime(t, f, nil) + p := r.Probe(testSessionID) + if !p.Exists || p.Dead || p.Attached != 2 || p.Activity.Unix() != 1700000000 { + t.Errorf("probe = %+v", p) + } + f.handler = func(args []string) ([]byte, error) { return []byte("1 0 5\n"), nil } + if p := r.Probe(testSessionID); !p.Dead { + t.Errorf("dead pane not reported: %+v", p) + } + f.handler = func(args []string) ([]byte, error) { return nil, errors.New("can't find session") } + if p := r.Probe(testSessionID); p.Exists { + t.Errorf("missing session reported as existing: %+v", p) + } + if p := r.Probe("not-an-id"); p.Exists { + t.Errorf("invalid id probed as existing") + } +} + +func TestUserSessionSendTextUsesPasteBufferNotKeyNames(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { + if args[0] == "display-message" { + return []byte("0 0 1700000000\n"), nil + } + return nil, nil + }} + r := newTestSessionRuntime(t, f, nil) + text := "Enter C-c\nline two; $(rm -rf /)" + if err := r.SendText(testSessionID, text, true); err != nil { + t.Fatal(err) + } + var load, paste, keys *fakeTmuxCall + for i := range f.calls { + switch f.calls[i].args[0] { + case "load-buffer": + load = &f.calls[i] + case "paste-buffer": + paste = &f.calls[i] + case "send-keys": + keys = &f.calls[i] + } + } + if load == nil || load.stdin != text { + t.Fatalf("text was not loaded verbatim via stdin: %+v", load) + } + if paste == nil || !strings.Contains(strings.Join(paste.args, " "), "-p") { + t.Fatalf("paste-buffer missing or not bracketed: %+v", paste) + } + for _, c := range f.calls { + if c.args[0] == "send-keys" && strings.Contains(strings.Join(c.args, " "), "C-c") { + t.Fatalf("user text reached send-keys as key names: %v", c.args) + } + } + if keys == nil || keys.args[len(keys.args)-1] != "Enter" { + t.Fatalf("submit did not press Enter: %+v", keys) + } + f.calls = nil + if err := r.SendText(testSessionID, "no submit", false); err != nil { + t.Fatal(err) + } + for _, v := range f.verbs() { + if v == "send-keys" { + t.Fatalf("Enter pressed without submit") + } + } +} + +func TestUserSessionSendTextRefusesDeadOrInvalid(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { return []byte("1 0 1700000000\n"), nil }} + r := newTestSessionRuntime(t, f, nil) + if err := r.SendText(testSessionID, "hi", false); err == nil { + t.Error("typing into a dead session succeeded") + } + f.handler = func(args []string) ([]byte, error) { return []byte("0 0 1700000000\n"), nil } + f.calls = nil + if err := r.SendText(testSessionID, "bad \x1b[201~ escape", false); err == nil { + t.Error("escape sequence accepted") + } + for _, v := range f.verbs() { + if v == "load-buffer" || v == "paste-buffer" { + t.Errorf("rejected text reached tmux: %v", f.verbs()) + } + } +} + +func TestSanitizeSessionInput(t *testing.T) { + ok := map[string]string{ + "plain": "plain", + "tab\tsep": "tab\tsep", + "a\r\nb": "a\nb", + "unicode: héllo ✓": "unicode: héllo ✓", + } + for in, want := range ok { + got, err := SanitizeSessionInput(in) + if err != nil || got != want { + t.Errorf("Sanitize(%q) = %q, %v; want %q", in, got, err, want) + } + } + for _, in := range []string{"\x1b[31m", "nul\x00", "bell\a", "del\x7f", "c1\u0085", "bad\xff"} { + if _, err := SanitizeSessionInput(in); err == nil { + t.Errorf("Sanitize(%q) accepted", in) + } + } +} + +func TestUserSessionKillIsIdempotent(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { return nil, errors.New("can't find session") }} + r := newTestSessionRuntime(t, f, nil) + if err := r.Kill(testSessionID); err != nil { + t.Errorf("kill of missing session = %v", err) + } + for _, v := range f.verbs() { + if v == "kill-session" { + t.Error("kill-session issued for a missing session") + } + } + f.handler = func(args []string) ([]byte, error) { return []byte("0 0 1\n"), nil } + f.calls = nil + if err := r.Kill(testSessionID); err != nil { + t.Fatal(err) + } + last := f.calls[len(f.calls)-1].args + if last[0] != "kill-session" || last[len(last)-1] != "hive-sess-"+testSessionID { + t.Errorf("kill args = %v", last) + } +} + +func TestUserSessionCaptureCapsLines(t *testing.T) { + f := &fakeTmux{handler: func(args []string) ([]byte, error) { return []byte("hello\n"), nil }} + r := newTestSessionRuntime(t, f, nil) + out, err := r.Capture(testSessionID, 0) + if err != nil || out != "hello\n" { + t.Fatalf("capture = %q, %v", out, err) + } + if !strings.Contains(strings.Join(f.calls[0].args, " "), "-S -50000") { + t.Errorf("uncapped capture args: %v", f.calls[0].args) + } + f.calls = nil + _, _ = r.Capture(testSessionID, 10_000_000) + if !strings.Contains(strings.Join(f.calls[0].args, " "), "-S -50000") { + t.Errorf("oversized capture not capped: %v", f.calls[0].args) + } + f.calls = nil + _, _ = r.Capture(testSessionID, 25) + if !strings.Contains(strings.Join(f.calls[0].args, " "), "-S -25 ") { + t.Errorf("requested line count not honoured: %v", f.calls[0].args) + } +} + +func TestUserSessionVendorsMatchKnownBackends(t *testing.T) { + const conf = "../../../config/backends.conf" + if _, err := os.Stat(conf); err != nil { + t.Skipf("backends.conf not reachable: %v", err) + } + out, err := exec.Command("bash", "-c", "source "+conf+" && printf '%s' \"$KNOWN_BACKENDS\"").Output() + if err != nil { + t.Fatalf("sourcing backends.conf: %v", err) + } + known := map[string]bool{} + for _, b := range strings.Fields(string(out)) { + known[b] = true + } + for _, v := range userSessionVendors { + if !known[v.id] { + t.Errorf("session vendor %q is not in KNOWN_BACKENDS", v.id) + } + if v.id == "bash" || v.id == "sh" || v.id == "shell" { + t.Errorf("shell vendor %q must never be offered", v.id) + } + } +} + +func TestUserSessionVendorsReportAvailability(t *testing.T) { + r := newTestSessionRuntime(t, &fakeTmux{}, nil) + r.lookPath = func(name string) (string, error) { + if name == "claude" { + return "/usr/bin/claude", nil + } + return "", errors.New("missing") + } + for _, v := range r.Vendors() { + if v.Available != (v.ID == "claude") { + t.Errorf("vendor %s available = %v", v.ID, v.Available) + } + } +} + +func TestUserSessionHomeBridgesVendorLoginsButNotGH(t *testing.T) { + newTestSessionRuntime(t, &fakeTmux{}, nil) + shared := sharedAgentHome + for _, p := range []string{ + filepath.Join(shared, ".claude", ".credentials.json"), + filepath.Join(shared, ".config", "gh", "hosts.yml"), + filepath.Join(shared, ".codex", "auth.json"), + } { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := setupUserSessionHome(testSessionID); err != nil { + t.Fatal(err) + } + home := userSessionHome(testSessionID) + if _, err := os.Lstat(filepath.Join(home, ".claude", ".credentials.json")); err != nil { + t.Errorf("claude login not bridged: %v", err) + } + if _, err := os.Lstat(filepath.Join(userSessionCodexHome(testSessionID), "auth.json")); err != nil { + t.Errorf("codex login not bridged: %v", err) + } + if _, err := os.Lstat(filepath.Join(home, ".config", "gh")); err == nil { + t.Error("gh config (hosts.yml token) was bridged into a session") + } +} diff --git a/src/crux/runtime/code_work.go b/src/crux/runtime/code_work.go index dd18b21eb9..86a5e8d24d 100644 --- a/src/crux/runtime/code_work.go +++ b/src/crux/runtime/code_work.go @@ -2,6 +2,7 @@ package runtime import ( "io" + "log/slog" "time" "github.com/hivecommons/hive/crux" @@ -14,3 +15,13 @@ import ( func ObserveCodeWorkTranscript(r io.Reader, workDir string, since, now time.Time) *crux.CodeWork { return agent.ObserveCodeWorkTranscript(r, workDir, since, now) } + +// UserSessions is the runtime behind the dashboard's Sessions feature +// (interactive, named CLI terminals). It is exposed here so consumers do not +// name the agent package. +type UserSessions = agent.UserSessionRuntime + +// NewUserSessions returns a user-session runtime driving the real tmux binary. +func NewUserSessions(logger *slog.Logger) *UserSessions { + return agent.NewUserSessionRuntime(logger) +} diff --git a/src/crux/usersession_types.go b/src/crux/usersession_types.go new file mode 100644 index 0000000000..9de9738597 --- /dev/null +++ b/src/crux/usersession_types.go @@ -0,0 +1,42 @@ +package crux + +import "time" + +// UserSessionVendor is one CLI a user session may run, as offered to the +// dashboard's new-session picker. +type UserSessionVendor struct { + ID string `json:"id"` + Label string `json:"label"` + // Available is true when the vendor's binary is on PATH. + Available bool `json:"available"` +} + +// UserSessionSpec is what the runtime needs to start (or resume) one user +// session. ID is the dashboard-issued session id (lower-case hex). +type UserSessionSpec struct { + ID string + Vendor string + // ResumeCommand, when set, is a backend resume command captured from the + // vendor's own transcript store (for example "claude --resume <id>"). The + // runtime re-validates it and runs it instead of the bare CLI. + ResumeCommand string +} + +// UserSessionProbe is one live read of a session's tmux state. +type UserSessionProbe struct { + // Exists is false when tmux has no such session. + Exists bool + // Dead is true when the session exists but its CLI has exited (the pane is + // kept so the scrollback stays readable). + Dead bool + // Attached is the number of tmux clients currently attached. + Attached int + // Activity is tmux's last-activity time for the session. + Activity time.Time +} + +// UserSessionResume names a vendor-native conversation a session can resume. +type UserSessionResume struct { + SessionID string + Command string +} diff --git a/src/deploy/ttyd-tmux.sh b/src/deploy/ttyd-tmux.sh index ac0c3abd6d..fd2b13beef 100644 --- a/src/deploy/ttyd-tmux.sh +++ b/src/deploy/ttyd-tmux.sh @@ -28,6 +28,19 @@ else SESSION_FROM_ARG=0 fi +# User sessions (dashboard Sessions feature): hive-sess-<id> runs on its OWN +# tmux server whose socket is named after the session. Reject any other +# hive-sess-* shape so ?arg= cannot reach a glob or a path. +USER_SESSION_RE='^hive-sess-[a-f0-9]{8,32}$' +case "$SESSION" in + hive-sess-*) + if ! [[ "$SESSION" =~ $USER_SESSION_RE ]]; then + echo "error: invalid user session name" >&2 + exit 1 + fi + ;; +esac + TTYD_WAIT_SEC="${HIVE_TTYD_WAIT_SEC:-900}" TTYD_POLL_SEC="${HIVE_TTYD_POLL_SEC:-2}" if ! [[ "$TTYD_WAIT_SEC" =~ ^[0-9]+$ ]]; then diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 33ea39c580..6b069a8b2a 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -22,6 +22,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Nous / strategy lab | [`api-reference/nous.md`](api-reference/nous.md) | 25 | | Inception | [`api-reference/inception.md`](api-reference/inception.md) | 14 | | Beads | [`api-reference/beads.md`](api-reference/beads.md) | 5 | +| Sessions | [`api-reference/sessions.md`](api-reference/sessions.md) | 8 | | Dashboard miscellaneous | [`api-reference/dashboard-misc.md`](api-reference/dashboard-misc.md) | 56 | | Hub SaaS | [`api-reference/hub-saas.md`](api-reference/hub-saas.md) | 102 | | Hub server | [`api-reference/hub-server.md`](api-reference/hub-server.md) | 46 | @@ -74,6 +75,10 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh 5 routes — see [api-reference/beads.md](api-reference/beads.md). +## Sessions + +8 routes — see [api-reference/sessions.md](api-reference/sessions.md). + ## Dashboard miscellaneous 56 routes — see [api-reference/dashboard-misc.md](api-reference/dashboard-misc.md). diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index 01bcaaa5f4..2fd867e7f9 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -23,20 +23,20 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:175` | | `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:176` | | `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:239` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:518` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:521` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:522` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:526` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:519` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:522` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:523` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:527` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:530` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:531` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:533` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:536` | -| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:537` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:323` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:324` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:325` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:326` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:531` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:532` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:534` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:537` | +| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:538` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:324` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:325` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:326` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:327` | | `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:126` | | `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:130` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 3213dc2ac3..d2c9ff2407 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1721` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1722` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1723` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1726` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1727` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1728` | diff --git a/src/docs/api-reference/beads.md b/src/docs/api-reference/beads.md index 443e2a5459..7733ba6f91 100644 --- a/src/docs/api-reference/beads.md +++ b/src/docs/api-reference/beads.md @@ -5,8 +5,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:502` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:503` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:504` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:505` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:506` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:503` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:504` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:505` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:506` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:507` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index a5a707e5e6..2f68b18a3c 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -71,26 +71,26 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `DELETE` | `/api/config/governor/gateways/{name}` | Owner only | Governor Gateways Delete | `pkg/dashboard/api.go:301` | | `POST` | `/api/config/governor/gateways/{name}/test` | Dashboard auth/session | Governor Gateways Test | `pkg/dashboard/api.go:302` | | `POST` | `/api/config/governor/gateways/discover` | Owner only | Governor Gateways Discover | `pkg/dashboard/api.go:303` | -| `POST` | `/api/config/governor/agents` | Owner only | Governor Add Agent | `pkg/dashboard/api.go:308` | -| `DELETE` | `/api/config/governor/agents/{name}` | Owner only | Governor Remove Agent | `pkg/dashboard/api.go:309` | -| `PUT` | `/api/config/governor/repos` | Owner only | Governor Repos | `pkg/dashboard/api.go:310` | -| `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:314` | -| `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:315` | -| `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:318` | +| `POST` | `/api/config/governor/agents` | Owner only | Governor Add Agent | `pkg/dashboard/api.go:309` | +| `DELETE` | `/api/config/governor/agents/{name}` | Owner only | Governor Remove Agent | `pkg/dashboard/api.go:310` | +| `PUT` | `/api/config/governor/repos` | Owner only | Governor Repos | `pkg/dashboard/api.go:311` | +| `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:315` | +| `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:316` | +| `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:319` | | `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:88` | | `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:89` | | `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:90` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:376` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:377` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:378` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:368` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:369` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:370` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:371` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:372` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:373` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:374` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:544` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:377` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:378` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:379` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:369` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:370` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:371` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:372` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:373` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:374` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:375` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:545` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:220` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:221` | @@ -99,8 +99,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:230` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:234` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:540` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:541` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:541` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:542` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:237` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:240` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:246` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 9009a479cf..fee159b157 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -25,7 +25,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](../audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:63` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:513` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:514` | | `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:66` | | `POST` | `/api/upgrade/announce` | Owner role | Announce a pod roll before it happens ([#15788](https://github.com/hivecommons/hive/issues/15788)): body `{"target": "<sha|tag>", "source": "hub"|"operator", "reason": "..."}` (`version` is an alias of `target`; all fields optional). Marks `/api/status`/`/api/version` `upgradeProgress` as `state: pulling, phase: announced` with the target and publishes an SSE `upgrading` event so every open dashboard shows the upgrade animation; a SIGTERM afterwards (or with no announce at all) publishes `phase: restarting` from the same record. The hub sends the same announce through the heartbeat response (`upgrade_announce`) for hub-managed hives; an announce with no kill expires after 15 minutes. Returns `{ok, upgradeProgress}` | `pkg/dashboard/api.go:69` | | `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:67` | @@ -40,18 +40,18 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:264` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:465` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:468` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:469` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:471` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:472` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:466` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:469` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:470` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:472` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:473` | | `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:215` | | `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:216` | | `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:217` | -| `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:306` | -| `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {<repo>: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:307` | +| `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:307` | +| `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {<repo>: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:308` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:551` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:552` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:118` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:123` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:137` | @@ -62,11 +62,11 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, `claimant: {kind: hive-agent|clanker|person, login, agent, hive, since, source: claim-ledger|claim-comment|assignee|kick-history|author}` naming who holds a claimed / in-progress issue or a PR under review or rework (absent when nobody is named; mirrored onto `linked_prs[]`), (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `decision_needed` on a row parked on a person (085 / 045 / 015, the needs-human pill; mirrored onto the issue's `linked_prs[]` entry): `{question, asked_by, since, blocker, blocker_reason, options: [{action, label, enabled, reason_if_disabled}]}` — the reviewer's own decision question when the verdict record has one, the blocker in plain words, and the applicable actions under the row context menu's action ids (`decide:delegate` first), (PRs) `lineage` on the hive's copy of an adopted PR — `{kind: adopted|superseded|rebased-by-hive, from_number, from_url, from_author, from_author_kind, to_number, to_url, at, reason}`, derived from the stage store's adoption record or the `hive/adopt-<n>` head branch, mirrored onto the issue's `linked_prs[]` entry; absent on every other row — and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close; a PR in a conflict-aware merge train carries `merge_train` `{id, state, front, position, size, attempt, max_attempts, overlap_files, ahead, length, head_number, head_state, head_reason, head_since, eta_s, eta_at, eta_basis, late, stalled, stalled_s}` — `head_state` one of `needs-human` / `ci` / `soaking` / `ready` / `merging` / `fixing` / `unknown` with `head_reason` in plain words; `eta_s` / `eta_at` the estimated wait for this member's own merge attempt (own remaining soak + each PR ahead's remaining soak or one sweep + the next sweep tick; absent while the head waits on a person or CI), `eta_at` anchored to the member's first estimate at its position so a countdown survives polls, `eta_basis` its short reading ("2 ahead · head soaking 12m · next sweep in 40s"), `late` once now has overrun the first estimate twice over, `stalled` / `stalled_s` once the head has sat longer than `governor.merge_train_stall_warn_s` ([#15395](https://github.com/hivecommons/hive/issues/15395)); a PR the queue holds because another open PR declares the same `Fixes #N` carries no `merge_train` and a `merge_gate` of `{state: "duplicate-of:<winner>", duplicate_of, duplicate_issue}` plus the `needs-decision` modifier (the queue keeps one PR per issue: green first, then the earlier-opened); a PR GitHub reports conflicting on its own, or one under repair (stage/070, 071, 072, 074, 085, or a red required check), carries no `merge_train` and is not counted in any member's `size` / `length` — a repaired member the view dropped carries `left_train: {why: fixing|needs-author|returned-to-contributor|adopted-by-hive|needs-decision|checks-failing, note}` instead (mirrored onto the issue's `linked_prs[]` entry), and a train left with one member reports no `merge_train` at all; a PR short of the merge queue (below stage/080) the planner ordered rides no train either — it carries `overlap: {behind, with, files, note}` ("overlaps #15637 (sidebar.html, index.html) — will queue behind it") and no `merge_train`, and counts in no member's `size` — the state slugs (`front`, `queued-behind`) are payload values only, the SPA renders ordinal places ("Queued for merge · 1st of 2", with the train's tag when more than one train is on the page). Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=<remaining>`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:191` | | `GET` | `/api/matrix/lines` | Dashboard auth/session | Matrix Lines (deprecated name, see [Published-checkpoint regions](#published-checkpoint-regions)): serve bounded backlog of pushed lines behind dashboard.matrix with repo allowlist filtering. `max` counts only lines the repo allowlist admits, so a page is never emptied by filtered-out lines. A client paging with `max` should use the last returned line's `seq` as its next `since`, not the top-level `seq`, which is the source's latest sequence number and would skip lines still past the page. `more` is true only while another line the caller may see remains past the page; stop paging when it is false. | `pkg/dashboard/api.go:192` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:193` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:381` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:444` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:445` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:474` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:508` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:382` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:445` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:446` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:475` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:509` | | `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:350` | | `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:351` | | `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:352` | @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1728` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1733` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 32337f311a..49e5b0e099 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1701` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1702` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1703` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1709` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1714` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1715` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1716` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1717` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1718` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1719` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1720` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1706` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1707` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1708` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1714` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1719` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1720` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1721` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1722` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1723` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1724` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1725` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/inception.md b/src/docs/api-reference/inception.md index 6da92d2d96..f31ef9698f 100644 --- a/src/docs/api-reference/inception.md +++ b/src/docs/api-reference/inception.md @@ -5,17 +5,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:447` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:448` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:449` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:450` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:451` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:452` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:453` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:454` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:455` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:456` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:457` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:458` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:459` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:460` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:448` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:449` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:450` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:451` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:452` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:453` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:454` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:455` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:456` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:457` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:458` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:459` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:460` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:461` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 08cafb1a1f..386cb49a09 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -5,38 +5,38 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:383` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:384` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:393` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:385` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:386` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:387` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:388` | -| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:390` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:394` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:395` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:396` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:397` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:398` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:399` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:400` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:401` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:407` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:408` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:409` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:410` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:411` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:412` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:413` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:414` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:415` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:416` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:417` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:418` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:419` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:420` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:421` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:422` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:384` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:385` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:394` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:386` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:387` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:388` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:389` | +| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:391` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:395` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:396` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:397` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:398` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:399` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:400` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:401` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:402` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:408` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:409` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:410` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:411` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:412` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:413` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:414` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:415` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:416` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:417` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:418` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:419` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:420` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:421` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:422` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:423` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -46,19 +46,19 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](../settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:432` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:433` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:434` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:435` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:436` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:437` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:438` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:439` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:440` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:441` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:442` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:405` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:406` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:433` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:434` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:435` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:436` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:437` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:438` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:439` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:440` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:441` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:442` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:443` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:406` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:407` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:25` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:26` | diff --git a/src/docs/api-reference/nous.md b/src/docs/api-reference/nous.md index 6400d1c981..0286d324e2 100644 --- a/src/docs/api-reference/nous.md +++ b/src/docs/api-reference/nous.md @@ -5,28 +5,28 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:476` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:477` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:478` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:479` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:480` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:481` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:482` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:483` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:484` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:485` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:486` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:487` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:488` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:489` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:490` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:491` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:492` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:493` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:494` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:495` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:496` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:497` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:498` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:499` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:500` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:477` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:478` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:479` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:480` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:481` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:482` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:483` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:484` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:485` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:486` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:487` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:488` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:489` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:490` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:491` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:492` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:493` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:494` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:495` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:496` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:497` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:498` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:499` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:500` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:501` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index b9bfed2203..7d2ae38d6d 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -5,29 +5,29 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:328` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:329` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:330` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:329` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:330` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:331` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](../repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:345`, `pkg/dashboard/api_repo_hold.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:346`, `pkg/dashboard/api_repo_hold.go` | | `GET` | `/api/repos/{owner}/{repo}/items/{number}` | Dashboard session | Per-item detail for the activity monitor's decision badge: `{repo, number, stage, stage_slug, operator_actions[], operator_action_count, last_operator_action}`. `operator_actions[]` is the server-side ledger of every row action submitted from the dashboard (hold/release, decision, decide-delegated, send-to-human, accept-triage — recorded by `recordOperatorAction` around those routes) as `{action, actor, at, note, target, outcome: pending|applied|no-op|failed, detail, head_sha, stage}`; `last_operator_action` adds the derived `state` (`waiting|addressed|stuck|pending|failed`), `stuck_after_seconds` and `followed[]` (the stage hops after the action). The same `last_operator_action` / `operator_action_count` ride every stage-tracker row and linked PR. Persisted in `/data/operator-actions.json`; `HIVE_OPERATOR_ACTION_STUCK_AFTER` (default `30m`) sets the stuck window. Rows also carry `rebase_merge: {state: approved|rebasing|ci-pending|merged|failed, actor, base, text, fail_reason, conflict_files[], armed_at}` while a *Rebase, then merge as maintainer* follow-through is live or just settled (persisted in `/data/rebase-merge-followups.json`; `fail_reason: foreign-head` when the head moved after the approval by someone other than the hive, so nothing merged), and `rebase: {attempt, max, exhausted, head, text}` on a conflicting PR the fixer is rebasing under the two-attempt loop guard (`/data/rebase-attempts.json`). | `pkg/dashboard/api.go`, `pkg/dashboard/operator_actions.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:349`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:350`, `pkg/dashboard/api_repo_item_decision.go` | -| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:354`, `pkg/dashboard/api_maintainer_notes.go` | -| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:355`, `pkg/dashboard/api_maintainer_notes.go` | -| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:350`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_repo_item_decision.go` | +| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:355`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_maintainer_notes.go` | +| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:357`, `pkg/dashboard/api_maintainer_notes.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `<!-- hive:delegated-decision … -->` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park, ladder?: {decision, reason, ruled_out[]}}, notes, error}`; `ladder` is set on contributor-PR decisions and names the rung that decided — `already-fixed` → `mergeable-as-is` → `rebase-then-merge` → `return-to-contributor` → `contributor-offline` — and why each rung above it did not apply); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:359`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:360` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:361` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:560`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:363`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:364` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:365` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:361` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:362` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:561`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:352`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:364`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:365` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:366` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:368`, `pkg/dashboard/api_hive_advice.go` | diff --git a/src/docs/api-reference/sessions.md b/src/docs/api-reference/sessions.md new file mode 100644 index 0000000000..df916aea30 --- /dev/null +++ b/src/docs/api-reference/sessions.md @@ -0,0 +1,28 @@ +# Dashboard REST API reference — Sessions + +Part of the [Dashboard REST API reference](../api-reference.md); see that page for how auth levels and `pkg/<file>:<line>` source citations are read and checked. Edit this file for routes in this area and run `src/scripts/check-api-reference-citations.sh --fix` to resync citations. + +Sessions are interactive, named CLI-agent terminals shared by everyone on a spoke. The feature is off unless `dashboard.sessions.enabled` (or `HIVE_SESSIONS_ENABLED`) is true; while off, `GET /api/sessions` answers `{"enabled":false,"sessions":[],"vendors":[],"limits":{}}` and every other route answers 404. Errors are `{"ok":false,"error":"..."}`. + +| Method | Path | Auth | Purpose | Source | +|---|---|---|---|---| +| `GET` | `/api/sessions` | Dashboard auth/session | Sessions, vendors and limits | `pkg/dashboard/api_sessions.go:99` | +| `POST` | `/api/sessions` | Owner or read-write | Create a session `{name, group, vendor}` (201) | `pkg/dashboard/api_sessions.go:100` | +| `PATCH` | `/api/sessions/{id}` | Session creator or owner | Rename or regroup `{name?, group?}` | `pkg/dashboard/api_sessions.go:101` | +| `DELETE` | `/api/sessions/{id}` | Session creator or owner | Close a running session (metadata kept, marked `ended`); on an ended session, remove its metadata | `pkg/dashboard/api_sessions.go:102` | +| `GET` | `/api/sessions/{id}/log` | Dashboard auth/session | Redacted scrollback `?lines=N` (capped at 5000) | `pkg/dashboard/api_sessions.go:103` | +| `POST` | `/api/sessions/{id}/input` | Owner or read-write | Type `{text, submit}` into the pane (size-capped, audited without the text) | `pkg/dashboard/api_sessions.go:104` | +| `POST` | `/api/sessions/{id}/upload` | Owner or read-write | Multipart `file`; returns the absolute path inside the session directory | `pkg/dashboard/api_sessions.go:105` | +| `POST` | `/api/sessions/{id}/resume` | Owner or read-write | Reopen an ended session's last conversation where the vendor supports it (409 otherwise) | `pkg/dashboard/api_sessions.go:106` | + +## Session object + +`{id, name, group, vendor, creator, created_at, last_activity, status, attached, tmux_name, resumable}` where `status` is `running` or `ended`, `attached` is the number of attached tmux clients and `tmux_name` is `hive-sess-<id>`. `resumable` is additive. + +## Attaching a terminal + +`POST /api/terminal/handoff` accepts an optional body `{"session":"<id>"}`. The response is unchanged (`{code, expires_in}`); the UI then opens `/terminal/?arg=hive-sess-<id>&code=<code>`. The terminal itself stays owner/read-write only. Keystrokes typed in the terminal are not attributable to a user; only `/input` and `/upload` are audited (`session_input`, `session_upload`, with user, session id and byte count, never the text). + +## Limits + +`limits` in `GET /api/sessions` reports `max_sessions`, `max_per_user`, `idle_ttl_hours`, `max_input_bytes`, `max_upload_bytes` and `max_uploads_per_session`. Defaults are 12, 4, 72 hours, 64 KiB, 10 MiB and 25; each is overridable (`HIVE_SESSIONS_MAX`, `HIVE_SESSIONS_MAX_PER_USER`, `HIVE_SESSIONS_IDLE_TTL_HOURS`, `HIVE_SESSIONS_MAX_INPUT_BYTES`, `HIVE_SESSIONS_MAX_UPLOAD_BYTES`, `HIVE_SESSIONS_MAX_UPLOADS`). diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index b657c1836b..b06253c0ef 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1791` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1796` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 6b846fe6ac..dbc283da19 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -228,6 +228,7 @@ new value at the same time. | `HIVE_DASHBOARD_UPGRADE_HELPER` | No | config `deployment.upgrade_helper` fallback, then `/usr/local/libexec/hive-dashboard-upgrade-helper` | Absolute path to the standalone upgrade helper executable the dashboard invokes. Must be a regular, executable file at an absolute path or the upgrade button stays hidden. See [dashboard-standalone-upgrades.md](dashboard-standalone-upgrades.md). | | `HIVE_GIT_BOT_EMAIL_DOMAIN` | No | `hive.kubestellar.io` | Domain of the bracket-free bot sign-off address `<slug>@<domain>`. With `github.app_signed_commits` on, the PR-request watcher re-addresses any `<slug>[bot]@users.noreply.github.com` `Signed-off-by` trailer it copies into the GitHub-signed commit to this form, because probot-dco rejects the bracketed local-part as a malformed email ([#6251](https://github.com/hivecommons/hive/issues/6251)); on v4 `src/deploy/entrypoint.sh` mints the agents' pane identity under the same domain ([#6276](https://github.com/hivecommons/hive/pull/6276)). A plain hostname only (`[A-Za-z0-9.-]`); anything else falls back to the default. | | `HIVE_TMUX_HISTORY_LIMIT` | No | `50000` | tmux scrollback depth applied when an agent session is created (positive integer; the authoritative knob for terminal scrollback and full-log capture). | +| `HIVE_SESSIONS_ENABLED` | No | config `dashboard.sessions.enabled` (default off) | Turns the dashboard Sessions feature (interactive CLI terminals, `/api/sessions`) on or off, overriding config. Limits: `HIVE_SESSIONS_MAX`, `HIVE_SESSIONS_MAX_PER_USER`, `HIVE_SESSIONS_IDLE_TTL_HOURS`; see [Sessions API](api-reference/sessions.md). | | `HIVE_TTYD_HISTORY_LIMIT` | No | `50000` | Defense-in-depth history-limit raise applied at browser attach time; only affects panes created after attach. | | `HIVE_TMUX_PANE_WIDTH` | No | `200` | Column count agent tmux sessions are created with. A detached tmux session defaults to 80 columns because no attached client supplies a size. | | `HIVE_KICK_LOG_DIR` | No | `/data/logs/kicks` | Root directory per-kick log archives are written under. On the persistent volume so archives survive restarts, pod rolls, and image upgrades. | diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 63002704d2..471bfa3f86 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -353,6 +353,21 @@ against `X-Hub-Signature-256`. Terminal access uses short-lived signed assertions. Systemd units refuse to run code from a directory an unprivileged user could have created. +**Sessions.** With `dashboard.sessions.enabled` (default off) the dashboard +offers interactive CLI terminals (`/api/sessions`, [API +reference](api-reference/sessions.md)). A session is a plain vendor CLI, never +a shell, launched with none of the permission-bypass flags agents run with, so +the CLI's own approval prompts stay on. Each session has its own tmux server, +HOME, `CODEX_HOME` and XDG directories, and its environment is built from an +allowlist: no dashboard token, terminal, hub, heartbeat or ttyd secret and no +GitHub token reaches it (only vendor login files are bridged from the shared +home, never `gh`'s `hosts.yml`). Creating, typing, uploading and attaching a +terminal need owner or read-write; the scrollback snapshot is readable by any +role and is token-redacted. Audit limits: `session_created`, `session_closed`, +`session_renamed`, `session_input` and `session_upload` record the user and +session id (never input text), but keystrokes typed inside the attached +terminal go through ttyd and are not attributable to a user. + **Dashboard headers.** Content Security Policy, `X-Frame-Options` and related headers are always set. The metrics endpoint is off unless `HIVE_METRICS_ENABLED` is set. diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 9486e24393..6d08dd957b 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1031,6 +1031,11 @@ dashboard: # authorized_users: [owner, reviewer:read, maintainer:read-write] # hub_proxied: false # True when trusted hub nginx injects identity headers # agent_poll_interval_s: 5 + # sessions: # Interactive named CLI terminals in the dashboard (default off) + # enabled: false # env HIVE_SESSIONS_ENABLED overrides; owner/read-write only, no bypass-permissions flags + # max_sessions: 12 # live sessions per spoke (env HIVE_SESSIONS_MAX) + # max_per_user: 4 # live sessions per creator (env HIVE_SESSIONS_MAX_PER_USER) + # idle_ttl_hours: 72 # close after this idle time (env HIVE_SESSIONS_IDLE_TTL_HOURS) # stage_tracker: # cache_ttl_s: 300 # Hive activity monitor payload cache; default = governor.eval_interval_s # theme: hive # Built-in theme id from /api/themes (for example hive, hive-dark, hive-light, star-wars, dungeons-and-dragons, star-trek, cyberpunk, terminal, solarized-dark, nord); or custom diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 82df5431be..4f60d58d69 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -5377,6 +5377,8 @@ type DashboardConfig struct { // StageTracker tunes the Hive activity monitor's server-side payload cache // (hivecommons/hive#12682). StageTracker DashboardStageTrackerConfig `yaml:"stage_tracker,omitempty" json:"stage_tracker,omitempty"` + // Sessions configures the interactive Sessions terminals (default off). + Sessions DashboardSessionsConfig `yaml:"sessions,omitempty" json:"sessions,omitempty"` } // DashboardStageTrackerConfig is dashboard.stage_tracker. diff --git a/src/pkg/config/dashboard_sessions.go b/src/pkg/config/dashboard_sessions.go new file mode 100644 index 0000000000..c25b15c853 --- /dev/null +++ b/src/pkg/config/dashboard_sessions.go @@ -0,0 +1,105 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// Defaults and bounds for the dashboard Sessions feature (interactive, named, +// shared CLI terminals). Every default is overridable by an env var so an +// operator can tune a spoke without editing hive.yaml. +const ( + // EnvSessionsEnabled overrides dashboard.sessions.enabled when set to a + // recognised boolean ("1/true/yes/on" or "0/false/no/off"). + EnvSessionsEnabled = "HIVE_SESSIONS_ENABLED" + // EnvSessionsMax, EnvSessionsMaxPerUser and EnvSessionsIdleTTLHours + // override the corresponding dashboard.sessions limits. + EnvSessionsMax = "HIVE_SESSIONS_MAX" + EnvSessionsMaxPerUser = "HIVE_SESSIONS_MAX_PER_USER" + EnvSessionsIdleTTLHours = "HIVE_SESSIONS_IDLE_TTL_HOURS" + + DefaultSessionsMax = 12 + DefaultSessionsMaxPerUser = 4 + DefaultSessionsIdleTTLHours = 72 +) + +// DashboardSessionsConfig is dashboard.sessions: named, shared CLI-agent +// terminal sessions inside the dashboard. Off by default. +type DashboardSessionsConfig struct { + // Enabled turns the Sessions feature on. HIVE_SESSIONS_ENABLED overrides it. + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // MaxSessions caps live (running) sessions per spoke. 0 = default (12). + MaxSessions int `yaml:"max_sessions,omitempty" json:"max_sessions,omitempty"` + // MaxPerUser caps live sessions one user may create. 0 = default (4). + MaxPerUser int `yaml:"max_per_user,omitempty" json:"max_per_user,omitempty"` + // IdleTTLHours closes a session with no input and no attached client for + // this many hours. 0 = default (72). + IdleTTLHours int `yaml:"idle_ttl_hours,omitempty" json:"idle_ttl_hours,omitempty"` +} + +// Validate rejects negative limits. +func (s DashboardSessionsConfig) Validate() error { + for name, v := range map[string]int{ + "max_sessions": s.MaxSessions, + "max_per_user": s.MaxPerUser, + "idle_ttl_hours": s.IdleTTLHours, + } { + if v < 0 { + return fmt.Errorf("dashboard.sessions.%s must not be negative (got %d)", name, v) + } + } + return nil +} + +// SessionsEnabled reports whether the Sessions feature is on: the +// HIVE_SESSIONS_ENABLED env wins over dashboard.sessions.enabled when it holds +// a recognised boolean. +func (c *Config) SessionsEnabled() bool { + if v, ok := parseBoolEnv(EnvSessionsEnabled); ok { + return v + } + return c != nil && c.Dashboard.Sessions.Enabled +} + +// SessionsMax returns the per-spoke live-session cap. +func (c *Config) SessionsMax() int { + var configured int + if c != nil { + configured = c.Dashboard.Sessions.MaxSessions + } + return sessionsLimit(EnvSessionsMax, configured, DefaultSessionsMax) +} + +// SessionsMaxPerUser returns the per-user live-session cap. +func (c *Config) SessionsMaxPerUser() int { + var configured int + if c != nil { + configured = c.Dashboard.Sessions.MaxPerUser + } + return sessionsLimit(EnvSessionsMaxPerUser, configured, DefaultSessionsMaxPerUser) +} + +// SessionsIdleTTLHours returns the idle-close threshold in hours. +func (c *Config) SessionsIdleTTLHours() int { + var configured int + if c != nil { + configured = c.Dashboard.Sessions.IdleTTLHours + } + return sessionsLimit(EnvSessionsIdleTTLHours, configured, DefaultSessionsIdleTTLHours) +} + +// sessionsLimit resolves env > config > default; non-positive values fall +// through to the next source. +func sessionsLimit(env string, configured, def int) int { + if v := strings.TrimSpace(os.Getenv(env)); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + if configured > 0 { + return configured + } + return def +} diff --git a/src/pkg/config/dashboard_sessions_test.go b/src/pkg/config/dashboard_sessions_test.go new file mode 100644 index 0000000000..8dd9387588 --- /dev/null +++ b/src/pkg/config/dashboard_sessions_test.go @@ -0,0 +1,67 @@ +package config + +import "testing" + +func TestSessionsEnabledEnvOverridesConfig(t *testing.T) { + c := &Config{} + t.Setenv(EnvSessionsEnabled, "") + if c.SessionsEnabled() { + t.Error("sessions enabled by default") + } + c.Dashboard.Sessions.Enabled = true + if !c.SessionsEnabled() { + t.Error("config flag ignored") + } + t.Setenv(EnvSessionsEnabled, "false") + if c.SessionsEnabled() { + t.Error("env false did not override config true") + } + c.Dashboard.Sessions.Enabled = false + t.Setenv(EnvSessionsEnabled, "1") + if !c.SessionsEnabled() { + t.Error("env true did not override config false") + } + t.Setenv(EnvSessionsEnabled, "garbage") + if c.SessionsEnabled() { + t.Error("unrecognised env value must fall back to config") + } + var nilCfg *Config + t.Setenv(EnvSessionsEnabled, "") + if nilCfg.SessionsEnabled() { + t.Error("nil config enabled sessions") + } +} + +func TestSessionsLimitsPrecedence(t *testing.T) { + for _, e := range []string{EnvSessionsMax, EnvSessionsMaxPerUser, EnvSessionsIdleTTLHours} { + t.Setenv(e, "") + } + c := &Config{} + if c.SessionsMax() != DefaultSessionsMax || c.SessionsMaxPerUser() != DefaultSessionsMaxPerUser || c.SessionsIdleTTLHours() != DefaultSessionsIdleTTLHours { + t.Errorf("defaults = %d/%d/%d", c.SessionsMax(), c.SessionsMaxPerUser(), c.SessionsIdleTTLHours()) + } + c.Dashboard.Sessions = DashboardSessionsConfig{MaxSessions: 5, MaxPerUser: 2, IdleTTLHours: 6} + if c.SessionsMax() != 5 || c.SessionsMaxPerUser() != 2 || c.SessionsIdleTTLHours() != 6 { + t.Errorf("config values ignored: %d/%d/%d", c.SessionsMax(), c.SessionsMaxPerUser(), c.SessionsIdleTTLHours()) + } + t.Setenv(EnvSessionsMax, "9") + t.Setenv(EnvSessionsMaxPerUser, "-3") + if c.SessionsMax() != 9 { + t.Errorf("env did not win: %d", c.SessionsMax()) + } + if c.SessionsMaxPerUser() != 2 { + t.Errorf("invalid env must fall through to config: %d", c.SessionsMaxPerUser()) + } +} + +func TestDashboardSessionsValidateRejectsNegative(t *testing.T) { + if err := (DashboardSessionsConfig{MaxSessions: -1}).Validate(); err == nil { + t.Error("negative max_sessions accepted") + } + if err := (DashboardSessionsConfig{IdleTTLHours: -1}).Validate(); err == nil { + t.Error("negative idle_ttl_hours accepted") + } + if err := (DashboardSessionsConfig{Enabled: true}).Validate(); err != nil { + t.Errorf("valid config rejected: %v", err) + } +} diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index f61ae46f36..1af875a147 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -474,7 +474,8 @@ } }, "issue_bands": {}, - "stage_tracker": {} + "stage_tracker": {}, + "sessions": {} }, "Data": { "MetricsDir": "/data/metrics", diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 1cc6845b18..95ecfd3857 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -262,6 +262,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.Dashboard.Matrix.Validate(); err != nil { return err } + if err := c.Dashboard.Sessions.Validate(); err != nil { + return err + } if normalized, err := ValidateDashboardPublicURL(c.Dashboard.PublicURL); err != nil { return err } else { diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index 0808453692..f06ffa5784 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -303,6 +303,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { s.mux.HandleFunc("POST /api/config/governor/gateways/discover", s.handleGovernorGatewaysDiscover) s.registerOpenRouterRoutes() s.registerLinearAgentRoutes() + s.registerSessionRoutes() s.mux.HandleFunc("GET /api/upstream-watch", s.handleUpstreamWatch) s.mux.HandleFunc("PUT /api/config/upstream-watch", s.handleUpstreamWatchConfigPut) s.mux.HandleFunc("POST /api/config/governor/agents", s.handleGovernorAddAgent) diff --git a/src/pkg/dashboard/api_sessions.go b/src/pkg/dashboard/api_sessions.go new file mode 100644 index 0000000000..5103b5f390 --- /dev/null +++ b/src/pkg/dashboard/api_sessions.go @@ -0,0 +1,723 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" +) + +// Sessions: interactive, named CLI terminals shared by everyone on a spoke. +// The tmux/ttyd engine sits behind SessionRuntime so it can be swapped; this +// file holds policy (roles, limits, validation, audit) and the JSON API. +// The terminal itself is attached through /terminal (terminal_handoff.go); +// keystrokes typed there are NOT attributable, only /input and /upload are. + +const ( + sessionsLogDefaultLines = 1000 + sessionsLogMaxLines = 5000 + // sessionsSweepInterval is how often idle sessions are checked. + sessionsSweepInterval = 5 * time.Minute + // sessionsJSONOverheadBytes is slack on top of the input cap for the JSON + // envelope (and escaping) when bounding the request body. + sessionsJSONOverheadBytes = 4096 + // sessionsMultipartOverheadBytes is slack on top of the upload cap for + // multipart framing. + sessionsMultipartOverheadBytes = 64 << 10 + // sessionsSmallBodyBytes bounds create/patch/handoff bodies. + sessionsSmallBodyBytes = 8 << 10 + + sessionsMaxInputBytesEnv = "HIVE_SESSIONS_MAX_INPUT_BYTES" + sessionsMaxUploadBytesEnv = "HIVE_SESSIONS_MAX_UPLOAD_BYTES" + sessionsMaxUploadsEnv = "HIVE_SESSIONS_MAX_UPLOADS" + + defaultSessionsMaxInputBytes = 64 << 10 + defaultSessionsMaxUploadBytes = 10 << 20 + defaultSessionsMaxUploads = 25 + + sessionUploadsDirName = "uploads" + sessionUploadDirMode = 0o700 + sessionUploadFileMode = 0o600 + sessionFileNameMax = 100 + sessionUploadFormName = "file" + + auditSessionCreated = "session_created" + auditSessionClosed = "session_closed" + auditSessionRenamed = "session_renamed" + auditSessionInput = "session_input" + auditSessionUpload = "session_upload" +) + +// SessionRuntime is the engine behind Sessions (implemented by +// crux/agent.UserSessionRuntime). Tests substitute a fake. +type SessionRuntime interface { + Vendors() []crux.UserSessionVendor + SupportsResume(vendor string) bool + WorkDir(id string) string + Start(spec crux.UserSessionSpec) error + Probe(id string) crux.UserSessionProbe + Kill(id string) error + Capture(id string, lines int) (string, error) + SendText(id, text string, submit bool) error + ResumeInfo(id, vendor string) (crux.UserSessionResume, bool) +} + +type sessionLimits struct { + MaxSessions int `json:"max_sessions"` + MaxPerUser int `json:"max_per_user"` + IdleTTLHours int `json:"idle_ttl_hours"` + MaxInputBytes int64 `json:"max_input_bytes"` + MaxUploadByte int64 `json:"max_upload_bytes"` + MaxUploads int `json:"max_uploads_per_session"` +} + +type sessionView struct { + ID string `json:"id"` + Name string `json:"name"` + Group string `json:"group"` + Vendor string `json:"vendor"` + Creator string `json:"creator"` + CreatedAt string `json:"created_at"` + LastActivity string `json:"last_activity"` + Status string `json:"status"` + Attached int `json:"attached"` + TmuxName string `json:"tmux_name"` + // Resumable (additive) is true for an ended session whose vendor can + // reopen its last conversation. + Resumable bool `json:"resumable"` +} + +func (s *Server) registerSessionRoutes() { + s.mux.HandleFunc("GET /api/sessions", s.handleSessionsList) + s.mux.HandleFunc("POST /api/sessions", s.handleSessionCreate) + s.mux.HandleFunc("PATCH /api/sessions/{id}", s.handleSessionPatch) + s.mux.HandleFunc("DELETE /api/sessions/{id}", s.handleSessionDelete) + s.mux.HandleFunc("GET /api/sessions/{id}/log", s.handleSessionLog) + s.mux.HandleFunc("POST /api/sessions/{id}/input", s.handleSessionInput) + s.mux.HandleFunc("POST /api/sessions/{id}/upload", s.handleSessionUpload) + s.mux.HandleFunc("POST /api/sessions/{id}/resume", s.handleSessionResume) +} + +// sessionsRuntime returns the runtime when the feature is enabled and wired. +func (s *Server) sessionsRuntime() (SessionRuntime, bool) { + if s.deps == nil || s.deps.Config == nil || !s.deps.Config.SessionsEnabled() || s.deps.Sessions == nil { + return nil, false + } + return s.deps.Sessions, true +} + +func (s *Server) sessionStoreOrNil() *sessionStore { + s.sessionsOnce.Do(func() { + dir := sessionsDir() + st, err := openSessionStore(dir) + if err != nil { + if s.logger != nil { + s.logger.Error("sessions store unreadable; starting empty and keeping the old file aside", "error", err) + } + _ = os.Rename(st.path, fmt.Sprintf("%s.corrupt-%d", st.path, time.Now().Unix())) + } + s.sessionStore = st + }) + return s.sessionStore +} + +func (s *Server) sessionLimits() sessionLimits { + cfg := s.deps.Config + return sessionLimits{ + MaxSessions: cfg.SessionsMax(), + MaxPerUser: cfg.SessionsMaxPerUser(), + IdleTTLHours: cfg.SessionsIdleTTLHours(), + MaxInputBytes: sessionEnvInt64(sessionsMaxInputBytesEnv, defaultSessionsMaxInputBytes), + MaxUploadByte: sessionEnvInt64(sessionsMaxUploadBytesEnv, defaultSessionsMaxUploadBytes), + MaxUploads: int(sessionEnvInt64(sessionsMaxUploadsEnv, defaultSessionsMaxUploads)), + } +} + +func sessionEnvInt64(name string, def int64) int64 { + if v := strings.TrimSpace(os.Getenv(name)); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return def +} + +// sessionActor classifies the caller. owner requires the server-verified +// owner marker (or a genuinely open spoke with no role at all); write is owner +// or read-write, the same set that may use the terminal. +type sessionActor struct { + user string + owner bool + write bool +} + +func (s *Server) sessionActorOf(r *http.Request) sessionActor { + role := strings.TrimSpace(r.Header.Get("X-Hive-Role")) + a := sessionActor{user: requestUser(r)} + switch { + case isOwnerRole(role): + a.owner = r.Header.Get(ownerRoleVerifiedHeader) == "true" + case role == "": + a.owner = s.requestRoleAllowsOwner(r) + } + a.write = a.owner || strings.EqualFold(role, config.RoleReadWrite) + return a +} + +func (s *Server) auditSession(user, action, detail string) { + if s.audit != nil { + s.audit.Log(user, action, detail, "") + } +} + +func (st sessionRecord) view(p *crux.UserSessionProbe, rt SessionRuntime) sessionView { + v := sessionView{ + ID: st.ID, Name: st.Name, Group: st.Group, Vendor: st.Vendor, Creator: st.Creator, + CreatedAt: st.CreatedAt, LastActivity: st.LastActivity, Status: st.Status, + TmuxName: "hive-sess-" + st.ID, + } + if st.Status == sessionStatusRunning && p != nil { + v.Attached = p.Attached + if !p.Activity.IsZero() { + if t, err := time.Parse(time.RFC3339, st.LastActivity); err != nil || p.Activity.After(t) { + v.LastActivity = p.Activity.UTC().Format(time.RFC3339) + } + } + } + v.Resumable = st.Status == sessionStatusEnded && st.ResumeCommand != "" && rt.SupportsResume(st.Vendor) + return v +} + +// reconcileSessions probes every running session once, marks those whose tmux +// session is gone (pod restart, CLI exit) as ended and returns the probes of +// the survivors. A restart of the hive process alone leaves tmux running, so +// those sessions stay running and reattach. +func (s *Server) reconcileSessions(rt SessionRuntime, st *sessionStore) map[string]crux.UserSessionProbe { + probes := map[string]crux.UserSessionProbe{} + now := time.Now().UTC().Format(time.RFC3339) + for _, rec := range st.list() { + if rec.Status != sessionStatusRunning { + continue + } + p := rt.Probe(rec.ID) + if p.Exists && !p.Dead { + probes[rec.ID] = p + continue + } + s.markSessionEnded(rt, st, rec, now) + } + return probes +} + +func (s *Server) markSessionEnded(rt SessionRuntime, st *sessionStore, rec sessionRecord, now string) { + resume := "" + if info, ok := rt.ResumeInfo(rec.ID, rec.Vendor); ok { + resume = info.Command + } + if _, _, err := st.update(rec.ID, func(r *sessionRecord) { + r.Status = sessionStatusEnded + r.EndedAt = now + if resume != "" { + r.ResumeCommand = resume + } + }); err != nil && s.logger != nil { + s.logger.Warn("sessions: persisting ended state failed", "id", rec.ID, "error", err) + } +} + +func (s *Server) handleSessionsList(w http.ResponseWriter, r *http.Request) { + rt, ok := s.sessionsRuntime() + if !ok { + jsonResponse(w, map[string]any{"enabled": false, "sessions": []sessionView{}, "vendors": []crux.UserSessionVendor{}, "limits": map[string]any{}}) + return + } + st := s.sessionStoreOrNil() + probes := s.reconcileSessions(rt, st) + views := []sessionView{} + for _, rec := range st.list() { + var p *crux.UserSessionProbe + if pr, ok := probes[rec.ID]; ok { + p = &pr + } + views = append(views, rec.view(p, rt)) + } + vendors := []crux.UserSessionVendor{} + for _, v := range rt.Vendors() { + if s.deps.Config.BackendAllowed(v.ID) { + vendors = append(vendors, v) + } + } + w.Header().Set("Cache-Control", "no-store") + jsonResponse(w, map[string]any{"enabled": true, "sessions": views, "vendors": vendors, "limits": s.sessionLimits()}) +} + +// sessionsGate answers 404 when the feature is off or unwired. +func (s *Server) sessionsGate(w http.ResponseWriter) (SessionRuntime, *sessionStore, bool) { + rt, ok := s.sessionsRuntime() + if !ok { + jsonError(w, "sessions are not enabled on this hive", http.StatusNotFound) + return nil, nil, false + } + return rt, s.sessionStoreOrNil(), true +} + +func readSmallJSON(w http.ResponseWriter, r *http.Request, v any) bool { + r.Body = http.MaxBytesReader(w, r.Body, sessionsSmallBodyBytes) + if err := json.NewDecoder(r.Body).Decode(v); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return false + } + return true +} + +func (s *Server) liveSessionCounts(st *sessionStore, user string) (total, mine int) { + for _, rec := range st.list() { + if rec.Status == sessionStatusRunning { + total++ + if rec.Creator == user { + mine++ + } + } + } + return total, mine +} + +// checkSessionCapacity answers 429 when a new live session would exceed a cap. +func (s *Server) checkSessionCapacity(w http.ResponseWriter, st *sessionStore, user string) bool { + lim := s.sessionLimits() + total, mine := s.liveSessionCounts(st, user) + switch { + case total >= lim.MaxSessions: + jsonError(w, fmt.Sprintf("this hive already has the maximum of %d live sessions", lim.MaxSessions), http.StatusTooManyRequests) + return false + case mine >= lim.MaxPerUser: + jsonError(w, fmt.Sprintf("you already have the maximum of %d live sessions", lim.MaxPerUser), http.StatusTooManyRequests) + return false + } + return true +} + +func (s *Server) handleSessionCreate(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + actor := s.sessionActorOf(r) + if !actor.write { + jsonError(w, "creating a session requires owner or read-write role", http.StatusForbidden) + return + } + var body struct { + Name string `json:"name"` + Group string `json:"group"` + Vendor string `json:"vendor"` + } + if !readSmallJSON(w, r, &body) { + return + } + body.Name, body.Group, body.Vendor = strings.TrimSpace(body.Name), strings.TrimSpace(body.Group), strings.TrimSpace(body.Vendor) + if err := validateSessionLabel("name", body.Name, sessionNameMaxRunes, true); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if err := validateSessionLabel("group", body.Group, sessionGroupMaxRunes, false); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + var vendor *crux.UserSessionVendor + for _, v := range rt.Vendors() { + if v.ID == body.Vendor { + v := v + vendor = &v + } + } + if vendor == nil { + jsonError(w, "unknown vendor", http.StatusBadRequest) + return + } + if !s.deps.Config.BackendAllowed(vendor.ID) { + jsonError(w, "this vendor is not allowed on this hive", http.StatusForbidden) + return + } + if !vendor.Available { + jsonError(w, vendor.Label+" is not installed on this hive", http.StatusConflict) + return + } + + s.sessionsMu.Lock() + defer s.sessionsMu.Unlock() + if !s.checkSessionCapacity(w, st, actor.user) { + return + } + id, err := st.newID() + if err != nil { + jsonError(w, "could not allocate a session id", http.StatusInternalServerError) + return + } + now := time.Now().UTC().Format(time.RFC3339) + rec := sessionRecord{ID: id, Name: body.Name, Group: body.Group, Vendor: vendor.ID, Creator: actor.user, CreatedAt: now, LastActivity: now, Status: sessionStatusRunning} + if err := rt.Start(crux.UserSessionSpec{ID: id, Vendor: vendor.ID}); err != nil { + if s.logger != nil { + s.logger.Warn("sessions: start failed", "id", id, "vendor", vendor.ID, "error", err) + } + jsonError(w, "could not start the session", http.StatusInternalServerError) + return + } + if err := st.put(rec); err != nil { + _ = rt.Kill(id) + jsonError(w, "could not save the session", http.StatusInternalServerError) + return + } + s.auditSession(actor.user, auditSessionCreated, auditDetail("session", id, "vendor", vendor.ID)) + jsonStatusResponse(w, http.StatusCreated, map[string]any{"ok": true, "session": rec.view(&crux.UserSessionProbe{}, rt)}) +} + +// sessionOwnedBy loads the record and enforces creator-or-owner. +func (s *Server) sessionForMutation(w http.ResponseWriter, r *http.Request, st *sessionStore) (sessionRecord, sessionActor, bool) { + rec, ok := st.get(r.PathValue("id")) + if !ok { + jsonError(w, "session not found", http.StatusNotFound) + return rec, sessionActor{}, false + } + actor := s.sessionActorOf(r) + if !actor.owner && !(actor.write && rec.Creator == actor.user) { + jsonError(w, "only the session's creator or an owner may change it", http.StatusForbidden) + return rec, actor, false + } + return rec, actor, true +} + +func (s *Server) handleSessionPatch(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + rec, actor, ok := s.sessionForMutation(w, r, st) + if !ok { + return + } + var body struct { + Name *string `json:"name"` + Group *string `json:"group"` + } + if !readSmallJSON(w, r, &body) { + return + } + if body.Name == nil && body.Group == nil { + jsonError(w, "nothing to change", http.StatusBadRequest) + return + } + name, group := rec.Name, rec.Group + if body.Name != nil { + name = strings.TrimSpace(*body.Name) + if err := validateSessionLabel("name", name, sessionNameMaxRunes, true); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + } + if body.Group != nil { + group = strings.TrimSpace(*body.Group) + if err := validateSessionLabel("group", group, sessionGroupMaxRunes, false); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + } + updated, _, err := st.update(rec.ID, func(x *sessionRecord) { x.Name, x.Group = name, group }) + if err != nil { + jsonError(w, "could not save the session", http.StatusInternalServerError) + return + } + s.auditSession(actor.user, auditSessionRenamed, auditDetail("session", rec.ID)) + p := rt.Probe(rec.ID) + jsonResponse(w, map[string]any{"ok": true, "session": updated.view(&p, rt)}) +} + +// handleSessionDelete closes a running session: tmux is killed and the +// metadata is kept, marked ended, so the session can be resumed where the +// vendor supports it. DELETE on an already ended session removes its metadata. +func (s *Server) handleSessionDelete(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + rec, actor, ok := s.sessionForMutation(w, r, st) + if !ok { + return + } + if rec.Status == sessionStatusEnded { + if err := rt.Kill(rec.ID); err != nil && s.logger != nil { + s.logger.Warn("sessions: killing ended session's leftover tmux failed", "id", rec.ID, "error", err) + } + if err := st.remove(rec.ID); err != nil { + jsonError(w, "could not save the session store", http.StatusInternalServerError) + return + } + s.auditSession(actor.user, auditSessionClosed, auditDetail("session", rec.ID, "removed", "true")) + jsonResponse(w, map[string]any{"ok": true}) + return + } + // Capture the resume handle before the tmux server goes away. + s.markSessionEnded(rt, st, rec, time.Now().UTC().Format(time.RFC3339)) + if err := rt.Kill(rec.ID); err != nil { + if s.logger != nil { + s.logger.Warn("sessions: kill failed", "id", rec.ID, "error", err) + } + jsonError(w, "could not close the session", http.StatusBadGateway) + return + } + s.auditSession(actor.user, auditSessionClosed, auditDetail("session", rec.ID)) + jsonResponse(w, map[string]any{"ok": true}) +} + +func (s *Server) handleSessionLog(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + rec, found := st.get(r.PathValue("id")) + if !found { + jsonError(w, "session not found", http.StatusNotFound) + return + } + n := sessionsLogDefaultLines + if v := r.URL.Query().Get("lines"); v != "" { + parsed, err := strconv.Atoi(v) + if err != nil || parsed <= 0 { + jsonError(w, "lines must be a positive integer", http.StatusBadRequest) + return + } + n = parsed + } + if n > sessionsLogMaxLines { + n = sessionsLogMaxLines + } + lines := []string{} + text, err := rt.Capture(rec.ID, n) + if err != nil { + if rec.Status == sessionStatusRunning { + jsonError(w, "could not read the session log", http.StatusBadGateway) + return + } + } else { + // Every pane surface redacts tokens before output leaves the server. + lines = strings.Split(strings.TrimRight(redactTokens(text), "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + } + w.Header().Set("Cache-Control", "no-store") + jsonResponse(w, map[string]any{"ok": true, "lines": lines, "status": rec.Status}) +} + +func (s *Server) handleSessionInput(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + actor := s.sessionActorOf(r) + if !actor.write { + jsonError(w, "sending input requires owner or read-write role", http.StatusForbidden) + return + } + rec, found := st.get(r.PathValue("id")) + if !found { + jsonError(w, "session not found", http.StatusNotFound) + return + } + if rec.Status != sessionStatusRunning { + jsonError(w, "session is not running", http.StatusConflict) + return + } + lim := s.sessionLimits() + r.Body = http.MaxBytesReader(w, r.Body, lim.MaxInputBytes+sessionsJSONOverheadBytes) + var body struct { + Text string `json:"text"` + Submit bool `json:"submit"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + jsonError(w, "input is too large", http.StatusRequestEntityTooLarge) + return + } + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + if int64(len(body.Text)) > lim.MaxInputBytes { + jsonError(w, fmt.Sprintf("input exceeds %d bytes", lim.MaxInputBytes), http.StatusRequestEntityTooLarge) + return + } + if body.Text == "" && !body.Submit { + jsonError(w, "nothing to send", http.StatusBadRequest) + return + } + if err := rt.SendText(rec.ID, body.Text, body.Submit); err != nil { + if strings.Contains(err.Error(), "control character") || strings.Contains(err.Error(), "UTF-8") { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + jsonError(w, "could not send input to the session", http.StatusBadGateway) + return + } + s.touchSession(st, rec.ID) + // Who typed, how much, and into which session; never the text itself. + s.auditSession(actor.user, auditSessionInput, auditDetail("session", rec.ID, "bytes", strconv.Itoa(len(body.Text)), "submit", strconv.FormatBool(body.Submit))) + jsonResponse(w, map[string]any{"ok": true}) +} + +func (s *Server) touchSession(st *sessionStore, id string) { + now := time.Now().UTC().Format(time.RFC3339) + _, _, _ = st.update(id, func(x *sessionRecord) { x.LastActivity = now }) +} + +func (s *Server) handleSessionResume(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + actor := s.sessionActorOf(r) + if !actor.write { + jsonError(w, "resuming a session requires owner or read-write role", http.StatusForbidden) + return + } + rec, found := st.get(r.PathValue("id")) + if !found { + jsonError(w, "session not found", http.StatusNotFound) + return + } + if rec.Status != sessionStatusEnded { + jsonError(w, "session is not ended", http.StatusConflict) + return + } + if !rt.SupportsResume(rec.Vendor) { + jsonError(w, "this vendor cannot resume a session", http.StatusConflict) + return + } + cmd := rec.ResumeCommand + if cmd == "" { + if info, ok := rt.ResumeInfo(rec.ID, rec.Vendor); ok { + cmd = info.Command + } + } + if cmd == "" { + jsonError(w, "no conversation to resume was found", http.StatusConflict) + return + } + if !s.deps.Config.BackendAllowed(rec.Vendor) { + jsonError(w, "this vendor is not allowed on this hive", http.StatusForbidden) + return + } + s.sessionsMu.Lock() + defer s.sessionsMu.Unlock() + if !s.checkSessionCapacity(w, st, actor.user) { + return + } + if err := rt.Start(crux.UserSessionSpec{ID: rec.ID, Vendor: rec.Vendor, ResumeCommand: cmd}); err != nil { + if s.logger != nil { + s.logger.Warn("sessions: resume failed", "id", rec.ID, "error", err) + } + jsonError(w, "could not resume the session", http.StatusInternalServerError) + return + } + now := time.Now().UTC().Format(time.RFC3339) + updated, _, err := st.update(rec.ID, func(x *sessionRecord) { + x.Status, x.EndedAt, x.LastActivity = sessionStatusRunning, "", now + }) + if err != nil { + jsonError(w, "could not save the session", http.StatusInternalServerError) + return + } + s.auditSession(actor.user, auditSessionCreated, auditDetail("session", rec.ID, "vendor", rec.Vendor, "resumed", "true")) + jsonResponse(w, map[string]any{"ok": true, "session": updated.view(&crux.UserSessionProbe{}, rt)}) +} + +// StartSessionSweeper reconciles sessions at start (hive restarted or the pod +// was replaced) and then closes idle ones every sessionsSweepInterval until ctx +// ends. It is a no-op loop while the feature is off. +func (s *Server) StartSessionSweeper(ctx context.Context) { + go func() { + s.sweepSessions(time.Now()) + t := time.NewTicker(sessionsSweepInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-t.C: + s.sweepSessions(now) + } + } + }() +} + +// sweepSessions marks vanished sessions ended, closes sessions idle past the +// TTL (no attached client and no activity), and prunes old ended metadata. +func (s *Server) sweepSessions(now time.Time) { + rt, ok := s.sessionsRuntime() + if !ok { + return + } + st := s.sessionStoreOrNil() + probes := s.reconcileSessions(rt, st) + ttl := time.Duration(s.deps.Config.SessionsIdleTTLHours()) * time.Hour + for _, rec := range st.list() { + switch rec.Status { + case sessionStatusRunning: + p, alive := probes[rec.ID] + if !alive || p.Attached > 0 { + continue + } + last, err := time.Parse(time.RFC3339, rec.LastActivity) + if err != nil { + continue + } + if p.Activity.After(last) { + last = p.Activity + } + if now.Sub(last) < ttl { + continue + } + s.markSessionEnded(rt, st, rec, now.UTC().Format(time.RFC3339)) + if err := rt.Kill(rec.ID); err != nil && s.logger != nil { + s.logger.Warn("sessions: idle kill failed", "id", rec.ID, "error", err) + } + s.auditSession("system", auditSessionClosed, auditDetail("session", rec.ID, "reason", "idle")) + case sessionStatusEnded: + if ended, err := time.Parse(time.RFC3339, rec.EndedAt); err == nil && now.Sub(ended) > sessionEndedRetention { + _ = st.remove(rec.ID) + } + } + } +} + +// sessionHandoffTarget validates a terminal handoff request for a session id: +// the feature must be on and the session must be running. It writes the error +// response itself. +func (s *Server) sessionHandoffTarget(w http.ResponseWriter, id string) bool { + rt, st, ok := s.sessionsGate(w) + if !ok { + return false + } + rec, found := st.get(id) + if !found { + jsonError(w, "session not found", http.StatusNotFound) + return false + } + if rec.Status != sessionStatusRunning { + jsonError(w, "session is not running", http.StatusConflict) + return false + } + if p := rt.Probe(id); !p.Exists { + jsonError(w, "session is not running", http.StatusConflict) + return false + } + return true +} diff --git a/src/pkg/dashboard/api_sessions_store.go b/src/pkg/dashboard/api_sessions_store.go new file mode 100644 index 0000000000..dfcfbb7bc5 --- /dev/null +++ b/src/pkg/dashboard/api_sessions_store.go @@ -0,0 +1,222 @@ +package dashboard + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + "unicode" + "unicode/utf8" +) + +// Sessions metadata store: /data/sessions/sessions.json (0600, atomic +// rename). The tmux processes do not survive a container restart; this file +// does, so ended sessions keep their name, group and resume handle. +const ( + sessionsDirEnv = "HIVE_SESSIONS_DIR" + defaultSessionsDir = "/data/sessions" + sessionsStoreFileName = "sessions.json" + sessionsStoreFileMode = 0o600 + sessionsStoreDirMode = 0o700 + + sessionStatusRunning = "running" + sessionStatusEnded = "ended" + + // sessionIDBytes random bytes -> 2x hex chars; matches the runtime's id + // pattern ([a-f0-9]{8,32}). + sessionIDRandomBytes = 6 + sessionNameMaxRunes = 64 + sessionGroupMaxRunes = 48 + // sessionEndedRetention is how long ended-session metadata is kept before + // the sweeper prunes it. + sessionEndedRetention = 30 * 24 * time.Hour +) + +// sessionRecord is the persisted form of one session. +type sessionRecord struct { + ID string `json:"id"` + Name string `json:"name"` + Group string `json:"group"` + Vendor string `json:"vendor"` + Creator string `json:"creator"` + CreatedAt string `json:"created_at"` + LastActivity string `json:"last_activity"` + Status string `json:"status"` + EndedAt string `json:"ended_at,omitempty"` + // ResumeCommand is the vendor-native resume command captured when the + // session ended. It is internal: never sent to the browser. + ResumeCommand string `json:"resume_command,omitempty"` +} + +type sessionStore struct { + mu sync.Mutex + path string + sessions map[string]*sessionRecord +} + +func sessionsDir() string { + if v := os.Getenv(sessionsDirEnv); v != "" { + return v + } + return defaultSessionsDir +} + +// openSessionStore loads the store at dir/sessions.json; a missing file is an +// empty store. +func openSessionStore(dir string) (*sessionStore, error) { + st := &sessionStore{path: filepath.Join(dir, sessionsStoreFileName), sessions: map[string]*sessionRecord{}} + data, err := os.ReadFile(st.path) + if errors.Is(err, os.ErrNotExist) { + return st, nil + } + if err != nil { + return st, fmt.Errorf("reading sessions store: %w", err) + } + var recs []*sessionRecord + if err := json.Unmarshal(data, &recs); err != nil { + return st, fmt.Errorf("parsing sessions store: %w", err) + } + for _, rec := range recs { + if rec != nil && rec.ID != "" { + st.sessions[rec.ID] = rec + } + } + return st, nil +} + +// saveLocked writes the store atomically. Callers hold st.mu. +func (st *sessionStore) saveLocked() error { + if err := os.MkdirAll(filepath.Dir(st.path), sessionsStoreDirMode); err != nil { + return err + } + recs := st.listLocked() + data, err := json.MarshalIndent(recs, "", " ") + if err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(st.path), ".sessions-*.tmp") + if err != nil { + return err + } + defer func() { _ = os.Remove(tmp.Name()) }() + if err := tmp.Chmod(sessionsStoreFileMode); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), st.path) +} + +// listLocked returns copies sorted by creation time then id. +func (st *sessionStore) listLocked() []sessionRecord { + out := make([]sessionRecord, 0, len(st.sessions)) + for _, rec := range st.sessions { + out = append(out, *rec) + } + sort.Slice(out, func(i, j int) bool { + if out[i].CreatedAt != out[j].CreatedAt { + return out[i].CreatedAt < out[j].CreatedAt + } + return out[i].ID < out[j].ID + }) + return out +} + +func (st *sessionStore) list() []sessionRecord { + st.mu.Lock() + defer st.mu.Unlock() + return st.listLocked() +} + +func (st *sessionStore) get(id string) (sessionRecord, bool) { + st.mu.Lock() + defer st.mu.Unlock() + rec, ok := st.sessions[id] + if !ok { + return sessionRecord{}, false + } + return *rec, true +} + +// put inserts or replaces a record and persists it. +func (st *sessionStore) put(rec sessionRecord) error { + st.mu.Lock() + defer st.mu.Unlock() + st.sessions[rec.ID] = &rec + return st.saveLocked() +} + +// update applies fn to the record under the lock and persists the result. +func (st *sessionStore) update(id string, fn func(*sessionRecord)) (sessionRecord, bool, error) { + st.mu.Lock() + defer st.mu.Unlock() + rec, ok := st.sessions[id] + if !ok { + return sessionRecord{}, false, nil + } + fn(rec) + return *rec, true, st.saveLocked() +} + +func (st *sessionStore) remove(id string) error { + st.mu.Lock() + defer st.mu.Unlock() + delete(st.sessions, id) + return st.saveLocked() +} + +// newSessionRecordID returns a fresh lower-case hex id not present in the store. +func (st *sessionStore) newID() (string, error) { + for i := 0; i < 8; i++ { + b := make([]byte, sessionIDRandomBytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + id := hex.EncodeToString(b) + st.mu.Lock() + _, taken := st.sessions[id] + st.mu.Unlock() + if !taken { + return id, nil + } + } + return "", errors.New("could not allocate a session id") +} + +// validateSessionLabel checks a session name or group: letters, digits, space +// and . _ - ( ) & : + / only, no leading or trailing space, bounded length. +func validateSessionLabel(field, v string, maxRunes int, required bool) error { + if v == "" { + if required { + return fmt.Errorf("%s is required", field) + } + return nil + } + if !utf8.ValidString(v) || utf8.RuneCountInString(v) > maxRunes { + return fmt.Errorf("%s must be at most %d characters", field, maxRunes) + } + if v[0] == ' ' || v[len(v)-1] == ' ' { + return fmt.Errorf("%s must not start or end with a space", field) + } + for _, r := range v { + switch { + case unicode.IsLetter(r), unicode.IsDigit(r): + case r == ' ', r == '.', r == '_', r == '-', r == '(', r == ')', r == '&', r == ':', r == '+', r == '/': + default: + return fmt.Errorf("%s contains a character that is not allowed", field) + } + } + return nil +} diff --git a/src/pkg/dashboard/api_sessions_test.go b/src/pkg/dashboard/api_sessions_test.go new file mode 100644 index 0000000000..96a636affd --- /dev/null +++ b/src/pkg/dashboard/api_sessions_test.go @@ -0,0 +1,885 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "errors" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/terminalassert" +) + +// fakeSessionRuntime is a deterministic SessionRuntime: no tmux, no processes. +type fakeSessionRuntime struct { + mu sync.Mutex + vendors []crux.UserSessionVendor + workDir string + alive map[string]crux.UserSessionProbe + started []crux.UserSessionSpec + killed []string + typed []string + submits []bool + capture string + resume map[string]crux.UserSessionResume + startFn func(crux.UserSessionSpec) error +} + +func newFakeSessionRuntime(t *testing.T) *fakeSessionRuntime { + t.Helper() + return &fakeSessionRuntime{ + vendors: []crux.UserSessionVendor{ + {ID: "claude", Label: "Claude Code", Available: true}, + {ID: "codex", Label: "Codex", Available: true}, + {ID: "goose", Label: "Goose", Available: false}, + }, + workDir: t.TempDir(), + alive: map[string]crux.UserSessionProbe{}, + resume: map[string]crux.UserSessionResume{}, + } +} + +func (f *fakeSessionRuntime) Vendors() []crux.UserSessionVendor { return f.vendors } +func (f *fakeSessionRuntime) SupportsResume(v string) bool { return v == "claude" } +func (f *fakeSessionRuntime) WorkDir(id string) string { return filepath.Join(f.workDir, id) } +func (f *fakeSessionRuntime) Start(spec crux.UserSessionSpec) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.startFn != nil { + if err := f.startFn(spec); err != nil { + return err + } + } + f.started = append(f.started, spec) + f.alive[spec.ID] = crux.UserSessionProbe{Exists: true, Activity: time.Now()} + return nil +} +func (f *fakeSessionRuntime) Probe(id string) crux.UserSessionProbe { + f.mu.Lock() + defer f.mu.Unlock() + return f.alive[id] +} +func (f *fakeSessionRuntime) Kill(id string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.killed = append(f.killed, id) + delete(f.alive, id) + return nil +} +func (f *fakeSessionRuntime) Capture(id string, lines int) (string, error) { + return f.capture, nil +} +func (f *fakeSessionRuntime) SendText(id, text string, submit bool) error { + f.mu.Lock() + defer f.mu.Unlock() + if strings.ContainsRune(text, 0x1b) { + return errors.New("input contains a control character") + } + f.typed = append(f.typed, text) + f.submits = append(f.submits, submit) + return nil +} +func (f *fakeSessionRuntime) ResumeInfo(id, vendor string) (crux.UserSessionResume, bool) { + r, ok := f.resume[id] + return r, ok +} + +type sessionTestEnv struct { + s *Server + rt *fakeSessionRuntime +} + +func newSessionTestEnv(t *testing.T, enabled bool) *sessionTestEnv { + t.Helper() + t.Setenv(sessionsDirEnv, t.TempDir()) + t.Setenv(config.EnvSessionsEnabled, "") + for _, e := range []string{config.EnvSessionsMax, config.EnvSessionsMaxPerUser, config.EnvSessionsIdleTTLHours, + sessionsMaxInputBytesEnv, sessionsMaxUploadBytesEnv, sessionsMaxUploadsEnv} { + t.Setenv(e, "") + } + rt := newFakeSessionRuntime(t) + cfg := &config.Config{} + cfg.Dashboard.Sessions.Enabled = enabled + s := NewServer(0, nil) + s.deps = &Dependencies{Config: cfg, Sessions: rt} + // Routes are registered by RegisterAPI in production; the tests register + // just the Sessions routes on the bare server. + s.registerSessionRoutes() + return &sessionTestEnv{s: s, rt: rt} +} + +type sessionCaller struct { + user, role string + verified bool +} + +var ( + callerOwner = sessionCaller{"olivia", config.RoleOwner, true} + callerOwner2 = sessionCaller{"omar", config.RoleOwner, true} + callerRW = sessionCaller{"rita", config.RoleReadWrite, false} + callerRW2 = sessionCaller{"ray", config.RoleReadWrite, false} + callerMerger = sessionCaller{"mia", config.RoleMerger, false} + callerRead = sessionCaller{"reed", config.RoleRead, false} + callerForgedOwn = sessionCaller{"mallory", config.RoleOwner, false} +) + +func (e *sessionTestEnv) do(t *testing.T, c sessionCaller, method, path string, body any) *httptest.ResponseRecorder { + t.Helper() + var rdr *bytes.Reader + switch b := body.(type) { + case nil: + rdr = bytes.NewReader(nil) + case []byte: + rdr = bytes.NewReader(b) + default: + raw, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + rdr = bytes.NewReader(raw) + } + req := httptest.NewRequest(method, path, rdr) + req.Header.Set("X-Hive-User", c.user) + req.Header.Set("X-Hive-Role", c.role) + if c.verified { + req.Header.Set(ownerRoleVerifiedHeader, "true") + } + rec := httptest.NewRecorder() + e.s.mux.ServeHTTP(rec, req) + return rec +} + +func decodeSessionJSON(t *testing.T, rec *httptest.ResponseRecorder) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &m); err != nil { + t.Fatalf("response is not JSON: %q (%v)", rec.Body.String(), err) + } + return m +} + +func (e *sessionTestEnv) create(t *testing.T, c sessionCaller, name string) string { + t.Helper() + rec := e.do(t, c, "POST", "/api/sessions", map[string]string{"name": name, "group": "rfc", "vendor": "claude"}) + if rec.Code != http.StatusCreated { + t.Fatalf("create = %d: %s", rec.Code, rec.Body.String()) + } + sess, _ := decodeSessionJSON(t, rec)["session"].(map[string]any) + id, _ := sess["id"].(string) + if id == "" { + t.Fatalf("no id in %s", rec.Body.String()) + } + return id +} + +func (e *sessionTestEnv) auditActions() []AuditEntry { return e.s.audit.Recent(0) } + +func TestSessionsDisabledReportsEmptyAndHidesRoutes(t *testing.T) { + e := newSessionTestEnv(t, false) + rec := e.do(t, callerRead, "GET", "/api/sessions", nil) + if rec.Code != http.StatusOK { + t.Fatalf("GET disabled = %d", rec.Code) + } + m := decodeSessionJSON(t, rec) + if m["enabled"] != false { + t.Errorf("enabled = %v", m["enabled"]) + } + if s, _ := m["sessions"].([]any); s == nil || len(s) != 0 { + t.Errorf("sessions = %v", m["sessions"]) + } + if v, _ := m["vendors"].([]any); v == nil || len(v) != 0 { + t.Errorf("vendors = %v", m["vendors"]) + } + if l, _ := m["limits"].(map[string]any); l == nil || len(l) != 0 { + t.Errorf("limits = %v", m["limits"]) + } + for _, tc := range []struct{ method, path string }{ + {"POST", "/api/sessions"}, {"PATCH", "/api/sessions/abc"}, {"DELETE", "/api/sessions/abc"}, + {"GET", "/api/sessions/abc/log"}, {"POST", "/api/sessions/abc/input"}, + {"POST", "/api/sessions/abc/upload"}, {"POST", "/api/sessions/abc/resume"}, + } { + if rec := e.do(t, callerOwner, tc.method, tc.path, map[string]string{"name": "x"}); rec.Code != http.StatusNotFound { + t.Errorf("%s %s while disabled = %d, want 404", tc.method, tc.path, rec.Code) + } + } + if len(e.rt.started) != 0 { + t.Error("disabled feature started a session") + } +} + +func TestSessionsEnvOverridesConfigFlag(t *testing.T) { + e := newSessionTestEnv(t, false) + t.Setenv(config.EnvSessionsEnabled, "true") + if m := decodeSessionJSON(t, e.do(t, callerRead, "GET", "/api/sessions", nil)); m["enabled"] != true { + t.Errorf("env true did not enable: %v", m) + } + e = newSessionTestEnv(t, true) + t.Setenv(config.EnvSessionsEnabled, "false") + if m := decodeSessionJSON(t, e.do(t, callerRead, "GET", "/api/sessions", nil)); m["enabled"] != false { + t.Errorf("env false did not disable: %v", m) + } +} + +func TestSessionsListShapeVendorsAndLimits(t *testing.T) { + e := newSessionTestEnv(t, true) + e.s.deps.Config.Backends.Deny = []string{"codex"} + rec := e.do(t, callerRead, "GET", "/api/sessions", nil) + if rec.Code != http.StatusOK { + t.Fatalf("list = %d", rec.Code) + } + m := decodeSessionJSON(t, rec) + if m["enabled"] != true { + t.Fatalf("enabled = %v", m["enabled"]) + } + var ids []string + for _, v := range m["vendors"].([]any) { + ids = append(ids, v.(map[string]any)["id"].(string)) + } + if strings.Join(ids, ",") != "claude,goose" { + t.Errorf("vendors = %v, want denied codex filtered out", ids) + } + lim := m["limits"].(map[string]any) + for _, k := range []string{"max_sessions", "max_per_user", "idle_ttl_hours", "max_input_bytes", "max_upload_bytes"} { + if _, ok := lim[k]; !ok { + t.Errorf("limits missing %s: %v", k, lim) + } + } +} + +func TestSessionsRoleMatrix(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerOwner, "owner session") + post := map[string]string{"name": "n", "group": "g", "vendor": "claude"} + + type row struct { + who sessionCaller + create, input, patch int // expected status codes + } + // Create/input need owner or read-write; patch needs creator or owner and + // every non-owner here is not the creator of the owner's session. + rows := []row{ + {callerOwner, 201, 200, 200}, + {callerRW, 201, 200, 403}, + {callerMerger, 403, 403, 403}, + {callerRead, 403, 403, 403}, + {callerForgedOwn, 403, 403, 403}, + } + for _, r := range rows { + if rec := e.do(t, r.who, "POST", "/api/sessions", post); rec.Code != r.create { + t.Errorf("%s create = %d, want %d (%s)", r.who.role, rec.Code, r.create, rec.Body.String()) + } + if rec := e.do(t, r.who, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": "hi", "submit": true}); rec.Code != r.input { + t.Errorf("%s input = %d, want %d", r.who.role, rec.Code, r.input) + } + if rec := e.do(t, r.who, "PATCH", "/api/sessions/"+id, map[string]string{"name": "renamed"}); rec.Code != r.patch { + t.Errorf("%s patch = %d, want %d", r.who.role, rec.Code, r.patch) + } + } + // Reads are open to every role, including read. + for _, who := range []sessionCaller{callerOwner, callerRW, callerMerger, callerRead} { + if rec := e.do(t, who, "GET", "/api/sessions", nil); rec.Code != http.StatusOK { + t.Errorf("%s list = %d", who.role, rec.Code) + } + if rec := e.do(t, who, "GET", "/api/sessions/"+id+"/log", nil); rec.Code != http.StatusOK { + t.Errorf("%s log = %d", who.role, rec.Code) + } + } + // A forged owner role without the verified marker cannot delete. + if rec := e.do(t, callerForgedOwn, "DELETE", "/api/sessions/"+id, nil); rec.Code != http.StatusForbidden { + t.Errorf("forged owner delete = %d", rec.Code) + } +} + +func TestSessionsCreatorOrOwnerMayRenameAndClose(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "ritas session") + if rec := e.do(t, callerRW2, "PATCH", "/api/sessions/"+id, map[string]string{"group": "x"}); rec.Code != http.StatusForbidden { + t.Errorf("other read-write patched = %d", rec.Code) + } + if rec := e.do(t, callerRW2, "DELETE", "/api/sessions/"+id, nil); rec.Code != http.StatusForbidden { + t.Errorf("other read-write closed = %d", rec.Code) + } + rec := e.do(t, callerRW, "PATCH", "/api/sessions/"+id, map[string]string{"name": "renamed", "group": "new"}) + if rec.Code != http.StatusOK { + t.Fatalf("creator patch = %d: %s", rec.Code, rec.Body.String()) + } + sess := decodeSessionJSON(t, rec)["session"].(map[string]any) + if sess["name"] != "renamed" || sess["group"] != "new" { + t.Errorf("session after patch = %v", sess) + } + if rec := e.do(t, callerOwner2, "DELETE", "/api/sessions/"+id, nil); rec.Code != http.StatusOK { + t.Errorf("owner close = %d", rec.Code) + } +} + +func TestSessionsCreateValidation(t *testing.T) { + e := newSessionTestEnv(t, true) + cases := []struct { + name string + body map[string]string + want int + }{ + {"empty name", map[string]string{"name": "", "vendor": "claude"}, 400}, + {"long name", map[string]string{"name": strings.Repeat("a", sessionNameMaxRunes+1), "vendor": "claude"}, 400}, + {"bad chars", map[string]string{"name": "x<script>", "vendor": "claude"}, 400}, + {"newline", map[string]string{"name": "a\nb", "vendor": "claude"}, 400}, + {"bad group", map[string]string{"name": "ok", "group": "g;rm", "vendor": "claude"}, 400}, + {"unknown vendor", map[string]string{"name": "ok", "vendor": "emacs"}, 400}, + {"shell vendor", map[string]string{"name": "ok", "vendor": "bash"}, 400}, + {"not installed", map[string]string{"name": "ok", "vendor": "goose"}, 409}, + {"ok", map[string]string{"name": "Good name (v2)", "group": "RFC-1", "vendor": "claude"}, 201}, + } + for _, tc := range cases { + if rec := e.do(t, callerOwner, "POST", "/api/sessions", tc.body); rec.Code != tc.want { + t.Errorf("%s = %d, want %d (%s)", tc.name, rec.Code, tc.want, rec.Body.String()) + } + } + if rec := e.do(t, callerOwner, "POST", "/api/sessions", []byte("{not json")); rec.Code != http.StatusBadRequest { + t.Errorf("malformed body = %d", rec.Code) + } + e.s.deps.Config.Backends.Deny = []string{"claude"} + if rec := e.do(t, callerOwner, "POST", "/api/sessions", map[string]string{"name": "x", "vendor": "claude"}); rec.Code != http.StatusForbidden { + t.Errorf("denied backend = %d, want 403", rec.Code) + } + if len(e.rt.started) != 1 { + t.Errorf("started %d sessions, want only the one valid create", len(e.rt.started)) + } + for _, spec := range e.rt.started { + if spec.ResumeCommand != "" { + t.Errorf("fresh create carried a resume command") + } + } +} + +func TestSessionsLimits(t *testing.T) { + e := newSessionTestEnv(t, true) + t.Setenv(config.EnvSessionsMaxPerUser, "2") + t.Setenv(config.EnvSessionsMax, "3") + e.create(t, callerRW, "a") + e.create(t, callerRW, "b") + rec := e.do(t, callerRW, "POST", "/api/sessions", map[string]string{"name": "c", "vendor": "claude"}) + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("per-user cap = %d, want 429", rec.Code) + } + if m := decodeSessionJSON(t, rec); m["ok"] != false || m["error"] == "" { + t.Errorf("429 body = %v", m) + } + e.create(t, callerRW2, "d") + if rec := e.do(t, callerOwner, "POST", "/api/sessions", map[string]string{"name": "e", "vendor": "claude"}); rec.Code != http.StatusTooManyRequests { + t.Errorf("per-spoke cap = %d, want 429", rec.Code) + } +} + +func TestSessionsStartFailureLeavesNoRecord(t *testing.T) { + e := newSessionTestEnv(t, true) + e.rt.startFn = func(crux.UserSessionSpec) error { return errors.New("secret internal detail") } + rec := e.do(t, callerOwner, "POST", "/api/sessions", map[string]string{"name": "x", "vendor": "claude"}) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("start failure = %d", rec.Code) + } + if strings.Contains(rec.Body.String(), "secret internal detail") { + t.Error("internal error detail leaked to the client") + } + if got := e.s.sessionStoreOrNil().list(); len(got) != 0 { + t.Errorf("records after failed start: %v", got) + } +} + +func TestSessionsInputCapsAndAudit(t *testing.T) { + e := newSessionTestEnv(t, true) + t.Setenv(sessionsMaxInputBytesEnv, "32") + id := e.create(t, callerRW, "typing") + secret := "my-secret-passphrase" + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": secret, "submit": true}); rec.Code != http.StatusOK { + t.Fatalf("input = %d: %s", rec.Code, rec.Body.String()) + } + if len(e.rt.typed) != 1 || e.rt.typed[0] != secret || !e.rt.submits[0] { + t.Errorf("typed = %v submits = %v", e.rt.typed, e.rt.submits) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": strings.Repeat("x", 33)}); rec.Code != http.StatusRequestEntityTooLarge { + t.Errorf("oversized input = %d, want 413", rec.Code) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": strings.Repeat("x", 100000)}); rec.Code != http.StatusRequestEntityTooLarge { + t.Errorf("body over the cap = %d, want 413", rec.Code) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": ""}); rec.Code != http.StatusBadRequest { + t.Errorf("empty input = %d, want 400", rec.Code) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": "a\x1b[2J"}); rec.Code != http.StatusBadRequest { + t.Errorf("escape input = %d, want 400", rec.Code) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/nope/input", map[string]any{"text": "x"}); rec.Code != http.StatusNotFound { + t.Errorf("unknown session input = %d", rec.Code) + } + var found bool + for _, a := range e.auditActions() { + if strings.Contains(a.Detail, secret) { + t.Errorf("audit entry leaked input text: %+v", a) + } + if a.Action == auditSessionInput { + found = true + if a.User != "rita" || !strings.Contains(a.Detail, "session="+id) || !strings.Contains(a.Detail, "bytes=20") { + t.Errorf("input audit = %+v", a) + } + } + } + if !found { + t.Error("no session_input audit entry") + } + // An ended session refuses input. + if rec := e.do(t, callerOwner, "DELETE", "/api/sessions/"+id, nil); rec.Code != http.StatusOK { + t.Fatalf("close = %d", rec.Code) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": "x"}); rec.Code != http.StatusConflict { + t.Errorf("input to ended session = %d, want 409", rec.Code) + } +} + +func multipartBody(t *testing.T, field, filename string, content []byte) (*bytes.Buffer, string) { + t.Helper() + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + fw, err := mw.CreateFormFile(field, filename) + if err != nil { + t.Fatal(err) + } + _, _ = fw.Write(content) + _ = mw.Close() + return &buf, mw.FormDataContentType() +} + +func (e *sessionTestEnv) upload(t *testing.T, c sessionCaller, id, field, filename string, content []byte) *httptest.ResponseRecorder { + t.Helper() + body, ctype := multipartBody(t, field, filename, content) + req := httptest.NewRequest("POST", "/api/sessions/"+id+"/upload", body) + req.Header.Set("Content-Type", ctype) + req.Header.Set("X-Hive-User", c.user) + req.Header.Set("X-Hive-Role", c.role) + if c.verified { + req.Header.Set(ownerRoleVerifiedHeader, "true") + } + rec := httptest.NewRecorder() + e.s.mux.ServeHTTP(rec, req) + return rec +} + +func TestSessionsUploadStoresSanitizedFileInsideSessionDir(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "files") + rec := e.upload(t, callerRW, id, "file", "../../etc/pass wd;.sh", []byte("echo hi")) + if rec.Code != http.StatusOK { + t.Fatalf("upload = %d: %s", rec.Code, rec.Body.String()) + } + path, _ := decodeSessionJSON(t, rec)["path"].(string) + dir := filepath.Join(e.rt.WorkDir(id), sessionUploadsDirName) + if !filepath.IsAbs(path) || filepath.Dir(path) != dir { + t.Fatalf("path = %q, want a file directly inside %q", path, dir) + } + base := filepath.Base(path) + if strings.ContainsAny(base, " ;/\\") || strings.Contains(base, "..") { + t.Errorf("unsanitized name %q", base) + } + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != sessionUploadFileMode { + t.Errorf("mode = %v, want 0600", st.Mode().Perm()) + } + if st.Mode().Perm()&0o111 != 0 { + t.Error("uploaded file is executable") + } + var found bool + for _, a := range e.auditActions() { + if a.Action == auditSessionUpload { + found = true + if a.User != "rita" || strings.Contains(a.Detail, "passwd") { + t.Errorf("upload audit = %+v", a) + } + } + } + if !found { + t.Error("no session_upload audit entry") + } +} + +func TestSessionsUploadRolesCapsAndFieldName(t *testing.T) { + e := newSessionTestEnv(t, true) + t.Setenv(sessionsMaxUploadBytesEnv, "10") + t.Setenv(sessionsMaxUploadsEnv, "2") + id := e.create(t, callerRW, "files") + for _, who := range []sessionCaller{callerRead, callerMerger, callerForgedOwn} { + if rec := e.upload(t, who, id, "file", "a.txt", []byte("x")); rec.Code != http.StatusForbidden { + t.Errorf("%s upload = %d, want 403", who.role, rec.Code) + } + } + if rec := e.upload(t, callerRW, id, "other", "a.txt", []byte("x")); rec.Code != http.StatusBadRequest { + t.Errorf("wrong field = %d, want 400", rec.Code) + } + if rec := e.upload(t, callerRW, id, "file", "big.bin", bytes.Repeat([]byte("x"), 11)); rec.Code != http.StatusRequestEntityTooLarge { + t.Errorf("oversize = %d, want 413", rec.Code) + } + if entries, _ := os.ReadDir(filepath.Join(e.rt.WorkDir(id), sessionUploadsDirName)); len(entries) != 0 { + t.Errorf("oversize upload left %d files behind", len(entries)) + } + if rec := e.upload(t, callerRW, id, "file", "huge.bin", bytes.Repeat([]byte("x"), 1<<20)); rec.Code != http.StatusRequestEntityTooLarge { + t.Errorf("body far over the cap = %d, want 413", rec.Code) + } + for i := 0; i < 2; i++ { + if rec := e.upload(t, callerRW, id, "file", "ok.txt", []byte("x")); rec.Code != http.StatusOK { + t.Fatalf("upload %d = %d: %s", i, rec.Code, rec.Body.String()) + } + } + if rec := e.upload(t, callerRW, id, "file", "third.txt", []byte("x")); rec.Code != http.StatusTooManyRequests { + t.Errorf("file-count cap = %d, want 429", rec.Code) + } +} + +func TestSessionsLogRedactsAndCaps(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerOwner, "log") + e.rt.capture = "line one\nghp_abcdefghijklmnopqrstuvwxyz0123456789 token\n" + strings.Repeat("filler\n", 9000) + rec := e.do(t, callerRead, "GET", "/api/sessions/"+id+"/log?lines=3", nil) + if rec.Code != http.StatusOK { + t.Fatalf("log = %d", rec.Code) + } + m := decodeSessionJSON(t, rec) + if lines := m["lines"].([]any); len(lines) != 3 { + t.Errorf("lines = %d, want 3", len(lines)) + } + if m["status"] != "running" || m["ok"] != true { + t.Errorf("log meta = %v", m) + } + rec = e.do(t, callerRead, "GET", "/api/sessions/"+id+"/log?lines=999999", nil) + if lines := decodeSessionJSON(t, rec)["lines"].([]any); len(lines) != sessionsLogMaxLines { + t.Errorf("uncapped log returned %d lines, want %d", len(lines), sessionsLogMaxLines) + } + e.rt.capture = "line one\nghp_abcdefghijklmnopqrstuvwxyz0123456789 token\n" + rec = e.do(t, callerRead, "GET", "/api/sessions/"+id+"/log", nil) + if strings.Contains(rec.Body.String(), "ghp_abcdefghijklmnopqrstuvwxyz0123456789") { + t.Errorf("log leaked a GitHub token: %s", rec.Body.String()) + } + if rec := e.do(t, callerRead, "GET", "/api/sessions/"+id+"/log?lines=abc", nil); rec.Code != http.StatusBadRequest { + t.Errorf("bad lines = %d", rec.Code) + } + if rec := e.do(t, callerRead, "GET", "/api/sessions/zzz/log", nil); rec.Code != http.StatusNotFound { + t.Errorf("unknown session log = %d", rec.Code) + } +} + +func TestSessionsCloseKeepsMetadataAndAudits(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "closing") + if rec := e.do(t, callerRW, "DELETE", "/api/sessions/"+id, nil); rec.Code != http.StatusOK { + t.Fatalf("close = %d", rec.Code) + } + if len(e.rt.killed) == 0 || e.rt.killed[0] != id { + t.Errorf("tmux not killed: %v", e.rt.killed) + } + rec, ok := e.s.sessionStoreOrNil().get(id) + if !ok || rec.Status != sessionStatusEnded { + t.Fatalf("record after close = %+v, %v", rec, ok) + } + // A second DELETE removes the metadata. + if rec := e.do(t, callerRW, "DELETE", "/api/sessions/"+id, nil); rec.Code != http.StatusOK { + t.Fatalf("second delete = %d", rec.Code) + } + if _, ok := e.s.sessionStoreOrNil().get(id); ok { + t.Error("metadata not removed by deleting an ended session") + } + want := map[string]bool{auditSessionCreated: false, auditSessionClosed: false} + for _, a := range e.auditActions() { + if _, ok := want[a.Action]; ok { + want[a.Action] = true + if a.User != "rita" || !strings.Contains(a.Detail, id) { + t.Errorf("audit %s = %+v", a.Action, a) + } + } + if !auditSensitiveActions[a.Action] && strings.HasPrefix(a.Action, "session_") { + t.Errorf("%s missing from the sensitive allowlist", a.Action) + } + } + for k, v := range want { + if !v { + t.Errorf("no %s audit entry", k) + } + } +} + +func TestSessionsRenameAudited(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "before") + e.do(t, callerRW, "PATCH", "/api/sessions/"+id, map[string]string{"name": "after"}) + for _, a := range e.auditActions() { + if a.Action == auditSessionRenamed && a.User == "rita" { + return + } + } + t.Error("no session_renamed audit entry") +} + +func TestSessionsResume(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "resumable") + e.rt.resume[id] = crux.UserSessionResume{SessionID: "abc", Command: "claude --resume abc"} + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/resume", nil); rec.Code != http.StatusConflict { + t.Errorf("resume of running session = %d, want 409", rec.Code) + } + e.do(t, callerRW, "DELETE", "/api/sessions/"+id, nil) + list := decodeSessionJSON(t, e.do(t, callerRead, "GET", "/api/sessions", nil)) + sess := list["sessions"].([]any)[0].(map[string]any) + if sess["status"] != "ended" || sess["resumable"] != true { + t.Errorf("ended session view = %v", sess) + } + if rec := e.do(t, callerRead, "POST", "/api/sessions/"+id+"/resume", nil); rec.Code != http.StatusForbidden { + t.Errorf("read resume = %d", rec.Code) + } + rec := e.do(t, callerRW, "POST", "/api/sessions/"+id+"/resume", nil) + if rec.Code != http.StatusOK { + t.Fatalf("resume = %d: %s", rec.Code, rec.Body.String()) + } + last := e.rt.started[len(e.rt.started)-1] + if last.ID != id || last.ResumeCommand != "claude --resume abc" { + t.Errorf("resume start = %+v", last) + } + if got := decodeSessionJSON(t, rec)["session"].(map[string]any)["status"]; got != "running" { + t.Errorf("status after resume = %v", got) + } + + // A vendor without resume support yields 409. + id2 := e.create(t, callerRW, "codex") + e.do(t, callerOwner, "PATCH", "/api/sessions/"+id2, map[string]string{"name": "codex"}) + if _, _, err := e.s.sessionStoreOrNil().update(id2, func(r *sessionRecord) { r.Vendor, r.Status = "codex", sessionStatusEnded }); err != nil { + t.Fatal(err) + } + if rec := e.do(t, callerRW, "POST", "/api/sessions/"+id2+"/resume", nil); rec.Code != http.StatusConflict { + t.Errorf("unsupported vendor resume = %d, want 409", rec.Code) + } +} + +func TestSessionsStoreRoundTripAndPermissions(t *testing.T) { + dir := t.TempDir() + st, err := openSessionStore(dir) + if err != nil { + t.Fatal(err) + } + rec := sessionRecord{ID: "aabbccddeeff", Name: "n", Group: "g", Vendor: "claude", Creator: "u", + CreatedAt: "2026-01-01T00:00:00Z", LastActivity: "2026-01-01T00:00:00Z", Status: sessionStatusRunning, ResumeCommand: "claude --resume x"} + if err := st.put(rec); err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(dir, sessionsStoreFileName)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != sessionsStoreFileMode { + t.Errorf("store mode = %v, want 0600", info.Mode().Perm()) + } + reopened, err := openSessionStore(dir) + if err != nil { + t.Fatal(err) + } + got, ok := reopened.get(rec.ID) + if !ok || got != rec { + t.Errorf("round trip = %+v, %v; want %+v", got, ok, rec) + } + leftovers, _ := filepath.Glob(filepath.Join(dir, ".sessions-*.tmp")) + if len(leftovers) != 0 { + t.Errorf("temp files left behind: %v", leftovers) + } + if _, err := os.ReadFile(filepath.Join(dir, "missing", sessionsStoreFileName)); err == nil { + t.Fatal("sanity") + } + empty, err := openSessionStore(filepath.Join(dir, "missing")) + if err != nil || len(empty.list()) != 0 { + t.Errorf("missing store = %v, %v", empty.list(), err) + } + if err := os.WriteFile(filepath.Join(dir, sessionsStoreFileName), []byte("{broken"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := openSessionStore(dir); err == nil { + t.Error("corrupt store did not report an error") + } +} + +func TestSessionsResumeCommandNeverExposed(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "x") + e.rt.resume[id] = crux.UserSessionResume{Command: "claude --resume abc-secret-id"} + e.do(t, callerRW, "DELETE", "/api/sessions/"+id, nil) + rec := e.do(t, callerRead, "GET", "/api/sessions", nil) + if strings.Contains(rec.Body.String(), "abc-secret-id") || strings.Contains(rec.Body.String(), "resume_command") { + t.Errorf("resume command leaked to the browser: %s", rec.Body.String()) + } +} + +func TestSessionsReconcileMarksVanishedSessionsEnded(t *testing.T) { + e := newSessionTestEnv(t, true) + live := e.create(t, callerRW, "live") + gone := e.create(t, callerRW, "gone") + delete(e.rt.alive, gone) // tmux did not survive the restart + list := decodeSessionJSON(t, e.do(t, callerRead, "GET", "/api/sessions", nil)) + status := map[string]string{} + for _, v := range list["sessions"].([]any) { + m := v.(map[string]any) + status[m["id"].(string)] = m["status"].(string) + } + if status[live] != "running" || status[gone] != "ended" { + t.Errorf("statuses = %v", status) + } + // And it persists. + if rec, _ := e.s.sessionStoreOrNil().get(gone); rec.Status != sessionStatusEnded || rec.EndedAt == "" { + t.Errorf("persisted = %+v", rec) + } +} + +func TestSessionsAttachedCountSurfaced(t *testing.T) { + e := newSessionTestEnv(t, true) + id := e.create(t, callerRW, "x") + e.rt.alive[id] = crux.UserSessionProbe{Exists: true, Attached: 3, Activity: time.Now()} + sess := decodeSessionJSON(t, e.do(t, callerRead, "GET", "/api/sessions", nil))["sessions"].([]any)[0].(map[string]any) + if sess["attached"] != float64(3) || sess["tmux_name"] != "hive-sess-"+id { + t.Errorf("session = %v", sess) + } + for _, k := range []string{"id", "name", "group", "vendor", "creator", "created_at", "last_activity", "status"} { + if _, ok := sess[k]; !ok { + t.Errorf("session missing %s", k) + } + } +} + +func TestSessionsIdleSweeper(t *testing.T) { + e := newSessionTestEnv(t, true) + idle := e.create(t, callerRW, "idle") + busy := e.create(t, callerRW, "attached") + old := time.Now().Add(-100 * time.Hour) + for _, id := range []string{idle, busy} { + if _, _, err := e.s.sessionStoreOrNil().update(id, func(r *sessionRecord) { r.LastActivity = old.UTC().Format(time.RFC3339) }); err != nil { + t.Fatal(err) + } + e.rt.alive[id] = crux.UserSessionProbe{Exists: true, Activity: old} + } + e.rt.alive[busy] = crux.UserSessionProbe{Exists: true, Attached: 1, Activity: old} + e.s.sweepSessions(time.Now()) + if rec, _ := e.s.sessionStoreOrNil().get(idle); rec.Status != sessionStatusEnded { + t.Errorf("idle session status = %s, want ended", rec.Status) + } + if rec, _ := e.s.sessionStoreOrNil().get(busy); rec.Status != sessionStatusRunning { + t.Errorf("attached session was closed: %s", rec.Status) + } + var closed bool + for _, a := range e.auditActions() { + if a.Action == auditSessionClosed && a.User == "system" && strings.Contains(a.Detail, "reason=idle") { + closed = true + } + } + if !closed { + t.Error("no idle session_closed audit entry") + } + // Ended metadata older than the retention window is pruned. + if _, _, err := e.s.sessionStoreOrNil().update(idle, func(r *sessionRecord) { + r.EndedAt = time.Now().Add(-2 * sessionEndedRetention).UTC().Format(time.RFC3339) + }); err != nil { + t.Fatal(err) + } + e.s.sweepSessions(time.Now()) + if _, ok := e.s.sessionStoreOrNil().get(idle); ok { + t.Error("expired ended session was not pruned") + } +} + +func TestSessionLabelValidation(t *testing.T) { + good := []string{"a", "My session", "RFC-12 (draft)", "héllo", "a/b", "x_y.z+1"} + for _, v := range good { + if err := validateSessionLabel("name", v, sessionNameMaxRunes, true); err != nil { + t.Errorf("%q rejected: %v", v, err) + } + } + bad := []string{" lead", "trail ", "a<b", "a\"b", "a\tb", "a\\b", "a`b", "a$b", "a;b", "a|b", "a*b"} + for _, v := range bad { + if err := validateSessionLabel("name", v, sessionNameMaxRunes, true); err == nil { + t.Errorf("%q accepted", v) + } + } + if err := validateSessionLabel("group", "", sessionGroupMaxRunes, false); err != nil { + t.Errorf("empty optional group rejected: %v", err) + } +} + +func TestSanitizeUploadName(t *testing.T) { + cases := map[string]string{ + "report.pdf": "report.pdf", + "../../etc/passwd": "passwd", + `C:\dir\file name.t`: "file_name.t", + ".hidden": "hidden", + "": "upload", + "..": "upload", + "a b;c.txt": "a_b_c.txt", + } + for in, want := range cases { + if got := sanitizeUploadName(in); got != want { + t.Errorf("sanitizeUploadName(%q) = %q, want %q", in, got, want) + } + } + if got := sanitizeUploadName(strings.Repeat("a", 500)); len(got) > sessionFileNameMax { + t.Errorf("name not bounded: %d", len(got)) + } +} + +// Terminal handoff with a session id. +func TestTerminalHandoffWithSession(t *testing.T) { + t.Setenv("HIVE_TERMINAL_KEY", "test-terminal-key-0123456789") + e := newSessionTestEnv(t, true) + e.s.deps.Config.HiveID = "hosted-alpha" + if terminalassert.SigningKey() == "" { + t.Skip("terminal signing key not derivable in this environment") + } + id := e.create(t, callerRW, "attach") + + post := func(c sessionCaller, body any) *httptest.ResponseRecorder { + var raw []byte + if body != nil { + raw, _ = json.Marshal(body) + } + req := httptest.NewRequest("POST", terminalHandoffPath, bytes.NewReader(raw)) + req.Header.Set("X-Hive-User", c.user) + req.Header.Set("X-Hive-Role", c.role) + rec := httptest.NewRecorder() + e.s.handleCreateTerminalHandoff(rec, req) + return rec + } + rec := post(callerRW, map[string]string{"session": id}) + if rec.Code != http.StatusOK { + t.Fatalf("handoff for session = %d: %s", rec.Code, rec.Body.String()) + } + m := decodeSessionJSON(t, rec) + if m["code"] == "" || m["expires_in"] != "60" { + t.Errorf("handoff response shape changed: %v", m) + } + if rec := post(callerRW, nil); rec.Code != http.StatusOK { + t.Errorf("handoff without body = %d, want unchanged behaviour", rec.Code) + } + if rec := post(callerRead, map[string]string{"session": id}); rec.Code != http.StatusForbidden { + t.Errorf("read role handoff = %d, want 403", rec.Code) + } + if rec := post(callerRW, map[string]string{"session": "ffffffffffff"}); rec.Code != http.StatusNotFound { + t.Errorf("unknown session handoff = %d, want 404", rec.Code) + } + e.do(t, callerRW, "DELETE", "/api/sessions/"+id, nil) + if rec := post(callerRW, map[string]string{"session": id}); rec.Code != http.StatusConflict { + t.Errorf("ended session handoff = %d, want 409", rec.Code) + } + e.s.deps.Config.Dashboard.Sessions.Enabled = false + if rec := post(callerRW, map[string]string{"session": id}); rec.Code != http.StatusNotFound { + t.Errorf("disabled handoff = %d, want 404", rec.Code) + } +} diff --git a/src/pkg/dashboard/api_sessions_upload.go b/src/pkg/dashboard/api_sessions_upload.go new file mode 100644 index 0000000000..6181c2795f --- /dev/null +++ b/src/pkg/dashboard/api_sessions_upload.go @@ -0,0 +1,134 @@ +package dashboard + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" +) + +const sessionUploadPrefixBytes = 4 + +// sanitizeUploadName reduces a client-supplied file name to a safe base name: +// no path, no leading dots, only [A-Za-z0-9._-], bounded length. +func sanitizeUploadName(name string) string { + name = strings.ReplaceAll(name, "\\", "/") + name = name[strings.LastIndex(name, "/")+1:] + var b strings.Builder + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + out := strings.TrimLeft(b.String(), ".") + if len(out) > sessionFileNameMax { + out = out[len(out)-sessionFileNameMax:] + } + if out == "" { + return "upload" + } + return out +} + +// handleSessionUpload stores one multipart file ("file") under the session's +// uploads directory and returns its absolute path; the UI then types that path +// into the session through /input. The file is written 0600 and never +// executed or interpreted by the hive. +func (s *Server) handleSessionUpload(w http.ResponseWriter, r *http.Request) { + rt, st, ok := s.sessionsGate(w) + if !ok { + return + } + actor := s.sessionActorOf(r) + if !actor.write { + jsonError(w, "uploading requires owner or read-write role", http.StatusForbidden) + return + } + rec, found := st.get(r.PathValue("id")) + if !found { + jsonError(w, "session not found", http.StatusNotFound) + return + } + if rec.Status != sessionStatusRunning { + jsonError(w, "session is not running", http.StatusConflict) + return + } + lim := s.sessionLimits() + r.Body = http.MaxBytesReader(w, r.Body, lim.MaxUploadByte+sessionsMultipartOverheadBytes) + mr, err := r.MultipartReader() + if err != nil { + jsonError(w, "expected a multipart upload", http.StatusBadRequest) + return + } + var part *multipart.Part + for { + p, err := mr.NextPart() + if err != nil { + if isUploadTooLarge(err) { + jsonError(w, "upload is too large", http.StatusRequestEntityTooLarge) + return + } + jsonError(w, "no file field in the upload", http.StatusBadRequest) + return + } + if p.FormName() == sessionUploadFormName && p.FileName() != "" { + part = p + break + } + } + dir := filepath.Join(rt.WorkDir(rec.ID), sessionUploadsDirName) + if err := os.MkdirAll(dir, sessionUploadDirMode); err != nil { + jsonError(w, "could not prepare the upload directory", http.StatusInternalServerError) + return + } + if entries, err := os.ReadDir(dir); err == nil && len(entries) >= lim.MaxUploads { + jsonError(w, "this session already has the maximum of "+strconv.Itoa(lim.MaxUploads)+" uploaded files", http.StatusTooManyRequests) + return + } + prefix := make([]byte, sessionUploadPrefixBytes) + if _, err := rand.Read(prefix); err != nil { + jsonError(w, "could not name the upload", http.StatusInternalServerError) + return + } + dest := filepath.Join(dir, hex.EncodeToString(prefix)+"-"+sanitizeUploadName(part.FileName())) + f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, sessionUploadFileMode) + if err != nil { + jsonError(w, "could not store the upload", http.StatusInternalServerError) + return + } + n, copyErr := io.Copy(f, io.LimitReader(part, lim.MaxUploadByte+1)) + closeErr := f.Close() + switch { + case copyErr != nil && isUploadTooLarge(copyErr), copyErr == nil && n > lim.MaxUploadByte: + _ = os.Remove(dest) + jsonError(w, fmt.Sprintf("upload exceeds %d bytes", lim.MaxUploadByte), http.StatusRequestEntityTooLarge) + return + case copyErr != nil, closeErr != nil: + _ = os.Remove(dest) + jsonError(w, "could not store the upload", http.StatusInternalServerError) + return + } + abs, err := filepath.Abs(dest) + if err != nil { + abs = dest + } + s.touchSession(st, rec.ID) + // Who uploaded and how much, into which session; the name is not recorded. + s.auditSession(actor.user, auditSessionUpload, auditDetail("session", rec.ID, "bytes", strconv.FormatInt(n, 10))) + jsonResponse(w, map[string]any{"ok": true, "path": abs}) +} + +func isUploadTooLarge(err error) bool { + var tooBig *http.MaxBytesError + return errors.As(err, &tooBig) +} diff --git a/src/pkg/dashboard/audit.go b/src/pkg/dashboard/audit.go index 3e4a1e1a52..6ab7db81e9 100644 --- a/src/pkg/dashboard/audit.go +++ b/src/pkg/dashboard/audit.go @@ -125,6 +125,11 @@ var auditSensitiveActions = map[string]bool{ "relay_task_revoked": true, "relay_token_minted": true, "run_stage_reset": true, + "session_closed": true, + "session_created": true, + "session_input": true, + "session_renamed": true, + "session_upload": true, "signed_commit_reauthored": true, "tool_approval": true, } diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index ce7a5f7edd..898cf30714 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -72,6 +72,9 @@ type Dependencies struct { // programs against the crux.Runtime surface, not *agent.Manager; tests that // need manager-only seams unwrap it with agentManager. AgentMgr crux.Runtime + // Sessions is the interactive-session runtime behind /api/sessions + // (dashboard.sessions.enabled). Nil keeps the feature unavailable. + Sessions SessionRuntime // CruxRuntimeHealth reports the agent runtime's reachability for // /api/health (hivecommons/hive#14138 stage 4). nil (in-process, or a // test without a runtime) leaves the field out of the body. diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 079050c4cf..748d82fbbf 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -420,6 +420,11 @@ type Server struct { // (see EnableSessionPersistence). Guarded by sessionMu. sessionStorePath string + // Sessions (api_sessions.go): lazily opened metadata store and the mutex + // serialising create/resume so the per-spoke and per-user caps hold. + sessionsOnce sync.Once + sessionStore *sessionStore + sessionsMu sync.Mutex terminalHandoffMu sync.Mutex terminalHandoffs map[string]terminalHandoff @@ -1830,6 +1835,8 @@ func (s *Server) Start() error { s.StartRebaseMergeFollowThrough(followCtx) // Hourly anonymized benchmark push to the hub (hivecommons/hive#17694). s.StartBenchmarksPublisher(followCtx) + // Sessions: reconcile after a restart, then close idle sessions. + s.StartSessionSweeper(followCtx) return srv.Serve(ln) } diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 6c298855c7..9568f96b5d 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -227,6 +227,9 @@ convergence.mutation.max_writers_per_target # boundary capacity tuning; yaml-onl dashboard.matrix.enabled # experimental default-off Matrix block; gates the header tab via /api/status, no Settings control (#15862, #16042) dashboard.matrix.exclude_paths[] # experimental default-off Matrix block; nothing reads it yet (#15862) dashboard.matrix.repos[] # experimental default-off Matrix block; nothing reads it yet (#15862) +dashboard.sessions.idle_ttl_hours # Sessions limit; hive.yaml/env only, Settings control pending +dashboard.sessions.max_per_user # Sessions limit; hive.yaml/env only, Settings control pending +dashboard.sessions.max_sessions # Sessions limit; hive.yaml/env only, Settings control pending debug.pprof_addr # loopback-only pprof bind address, operator/debug knob, not a Settings control (#14681) dedupe.advisory_autoclose # advisory-checklist sweep; yaml-only (#15754) deployment.runtime # set via HIVE_DEPLOYMENT_RUNTIME env var or auto-detected, not a Settings control (#14072) diff --git a/src/pkg/dashboard/terminal_handoff.go b/src/pkg/dashboard/terminal_handoff.go index 557675d169..de4943a9a7 100644 --- a/src/pkg/dashboard/terminal_handoff.go +++ b/src/pkg/dashboard/terminal_handoff.go @@ -1,6 +1,7 @@ package dashboard import ( + "encoding/json" "net/http" "net/url" "strings" @@ -43,6 +44,14 @@ func (s *Server) handleCreateTerminalHandoff(w http.ResponseWriter, r *http.Requ jsonStatusResponse(w, http.StatusForbidden, map[string]string{"error": "terminal access requires owner or read-write role"}) return } + // Optional body {"session":"<id>"} attaches a user session (tmux + // hive-sess-<id>); an absent body keeps the agent-terminal behaviour. The + // minted code is unchanged: the UI opens /terminal/?arg=hive-sess-<id>. + if sessionID, ok := readHandoffSession(w, r); !ok { + return + } else if sessionID != "" && !s.sessionHandoffTarget(w, sessionID) { + return + } user := requestUser(r) if !s.canMintTerminalAssertion() { jsonStatusResponse(w, http.StatusServiceUnavailable, map[string]string{"error": "terminal handoff requires terminal signing key and hive id"}) @@ -57,6 +66,26 @@ func (s *Server) handleCreateTerminalHandoff(w http.ResponseWriter, r *http.Requ jsonResponse(w, map[string]string{"code": code, "expires_in": "60"}) } +// terminalHandoffBodyMax bounds the optional handoff request body. +const terminalHandoffBodyMax = 1024 + +// readHandoffSession reads the optional {"session": id} body. A missing or +// unparseable body means "no session": the minted code is identical either way, +// so legacy callers that send some other body keep working. +func readHandoffSession(w http.ResponseWriter, r *http.Request) (id string, ok bool) { + if r.Body == nil || r.ContentLength == 0 { + return "", true + } + var body struct { + Session string `json:"session"` + } + r.Body = http.MaxBytesReader(w, r.Body, terminalHandoffBodyMax) + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + return "", true + } + return strings.TrimSpace(body.Session), true +} + func (s *Server) canMintTerminalAssertion() bool { if terminalassert.SigningKey() == "" { return false From 77003339315e2a8fd618f8d6bc8dbebb08c4c2b0 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 18:37:44 -0500 Subject: [PATCH 184/223] =?UTF-8?q?=F0=9F=8C=B1=20dashboard:=20move=20the?= =?UTF-8?q?=20shared=20auth-headers=20helper=20and=20Campaigns/Jam=20UI=20?= =?UTF-8?q?out=20of=20inception.js=20(#18094)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 dashboard: move the shared auth-headers helper and the Campaigns/Jam UI out of inception.js Pure move, no logic change: _inceptionAuthHeaders() (used by ~24 other files) goes to js/auth-headers.js, and the Campaigns and Jam UI goes to js/campaigns.js, so both survive the Inception wizard UI being replaced by the Sessions section. static/index.html regenerated. Signed-off-by: Andy Anderson <andy@clubanderson.com> * ✨ dashboard: Sessions section (tabbed shared CLI sessions) replaces the Inception UI when enabled (#18098) Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 dashboard: Sessions help text meets the section help rules TestDashboardSectionHelpCoverage requires the mark text to equal the first line of the help-page entry, stay within 25 words and avoid the Hive term "agent". Same wording in both places, with the Inception note moved to the following paragraph. Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 dashboard: use a single-codepoint icon for the Sessions nav entry TestSidebarNavIconsAreSingleCodepoint rejects the desktop-computer emoji (glyph plus variation selector). Use the laptop emoji in the sidebar entry, the section header and the card title so TestSidebarNavIconsMatchSectionHeaders still agrees. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- changelog.d/added-sessions-ui.md | 1 + src/docs/dashboard-sections.md | 8 + .../dashboard-sections/workflow-and-agents.md | 18 +- src/docs/dashboard-source-layout.md | 3 +- src/docs/inception.md | 2 + .../dashboard_section_reorder_9062_test.go | 10 +- .../dashboard_section_summaries_test.go | 2 +- src/pkg/dashboard/sessions_ui.test.js | 108 +++ src/pkg/dashboard/static-src/css/sessions.css | 50 + src/pkg/dashboard/static-src/html/faq.html | 5 + ...nous-inception-knowledge-contributors.html | 7 + .../dashboard/static-src/html/sidebar.html | 1 + .../dashboard/static-src/js/agent-detail.js | 2 + .../dashboard/static-src/js/auth-headers.js | 9 + src/pkg/dashboard/static-src/js/campaigns.js | 390 ++++++++ src/pkg/dashboard/static-src/js/inception.js | 392 -------- src/pkg/dashboard/static-src/js/layout.js | 7 +- src/pkg/dashboard/static-src/js/refresh.js | 2 +- src/pkg/dashboard/static-src/js/sections.js | 6 +- src/pkg/dashboard/static-src/js/sessions.js | 624 +++++++++++++ src/pkg/dashboard/static-src/manifest.txt | 4 + src/pkg/dashboard/static/index.html | 855 ++++++++++++++++-- .../dashboard/webstatic/static_index_test.go | 2 +- 23 files changed, 2025 insertions(+), 483 deletions(-) create mode 100644 changelog.d/added-sessions-ui.md create mode 100644 src/pkg/dashboard/sessions_ui.test.js create mode 100644 src/pkg/dashboard/static-src/css/sessions.css create mode 100644 src/pkg/dashboard/static-src/js/auth-headers.js create mode 100644 src/pkg/dashboard/static-src/js/campaigns.js create mode 100644 src/pkg/dashboard/static-src/js/sessions.js diff --git a/changelog.d/added-sessions-ui.md b/changelog.d/added-sessions-ui.md new file mode 100644 index 0000000000..3586fd8378 --- /dev/null +++ b/changelog.d/added-sessions-ui.md @@ -0,0 +1 @@ +- Dashboard: a new Sessions section (when sessions are enabled) with tabbed, named, grouped, shared CLI terminals, a send/attach input bar and a read-only scrollback view for the read role; it replaces the Inception wizard UI and keeps Campaigns and Jam on a tab. diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index 22b306f356..d4e3b7902b 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -178,6 +178,14 @@ A step-by-step guide that turns a new project idea into a first set of files and Full entry: [Inception](dashboard-sections/workflow-and-agents.md#inception). +## Sessions + +Named terminal sessions running command-line AI tools inside Hive. Everyone on this hive can watch, and people with write access can type. + +Sessions are named and grouped, and the section replaces Inception when sessions are enabled. + +Full entry: [Sessions](dashboard-sections/workflow-and-agents.md#sessions). + ## Knowledge The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one. diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index bb9ec25b24..c79e55258e 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -186,7 +186,23 @@ A step-by-step guide that turns a new project idea into a first set of files and **When it appears.** Always. -**Settings that change it.** No settings change this section. See [Inception](../inception.md). +**Settings that change it.** No settings change this section. See [Inception](../inception.md). When `dashboard.sessions.enabled` is on, the Sessions section replaces it. + +## Sessions + +Named terminal sessions where a command-line AI agent runs inside Hive; everyone on the hive can watch, and people with write access can join in. + +**What it tells you.** A tab strip lists each session, grouped under headings you choose, with the CLI it runs and how many browsers are attached. A **Campaigns** tab keeps the campaigns and Jam view that used to live under Inception. The selected tab shows a live terminal; ended sessions show their last output and a **Resume** button. + +**How the numbers are worked out.** The attached count is the number of browsers currently attached to that terminal. Everyone attached shares one keyboard, so typing in the terminal is not attributed to a person. Text sent from the input bar under the terminal (paste, send, send and run, attach file) goes through the Sessions API and is attributed to you in the audit log. + +**What it is good for.** Use it to run a CLI agent without leaving the dashboard, hand a long task to a teammate by sharing a tab, or let someone who logs in later read the scrollback. People with the `read` role see a read-only scrollback that refreshes every few seconds instead of a terminal. Only `owner` and `read-write` roles can start sessions, type or upload. + +**Example.** You start a session named "fix flaky test" in the group "ci" with the Claude CLI, paste the failing log into the input bar and press Send and run. A teammate opens the same tab and watches. + +**When it appears.** Only when sessions are enabled on the hive (`GET /api/sessions` reports `enabled: true`); otherwise the Inception section is shown. + +**Settings that change it.** `dashboard.sessions.enabled` (default off) and the session limits it reports. See [Inception](../inception.md) for the section it replaces. ## Knowledge diff --git a/src/docs/dashboard-source-layout.md b/src/docs/dashboard-source-layout.md index 2e907c305a..434fcf723c 100644 --- a/src/docs/dashboard-source-layout.md +++ b/src/docs/dashboard-source-layout.md @@ -221,7 +221,8 @@ JS (`js/`): `boot`, `core`, `feedback`, `sections`, `layout`, `agent-detail`, `governor-view`, `repos`, `overview`, `label-filter`, `pinned-filters`, `repos-render`, `budget`, `backends-tokens`, `cost`, `acmm-lifecycle`, `throughput`, `evidence`, `approvals`, `nous`, `knowledge`, `vault-audit`, -`review-queue`, `contributors`, `refresh`, `inception`, `debug-logs`, +`review-queue`, `contributors`, `refresh`, `auth-headers`, `campaigns`, +`inception`, `debug-logs`, `tooltips-dialogs`, `plan-agent-controls`, `acmm`, `version`, `sse-bootstrap`, `config-state`, `settings-search`, `config-dialog`, `config-tabs`, `config-webhooks`, `config-hub`, `config-agent`, `config-governor`, `config-watchdog-security`, diff --git a/src/docs/inception.md b/src/docs/inception.md index 28357b1d0d..a06fa2d270 100644 --- a/src/docs/inception.md +++ b/src/docs/inception.md @@ -10,6 +10,8 @@ This guide documents only what the code in `pkg/dashboard/inception_handlers.go` and `pkg/knowledge/inception.go` actually does — see those files for implementation detail. +> **Dashboard UI.** When `dashboard.sessions.enabled` is on, the dashboard hides the Inception wizard and shows the Sessions section instead (the Campaigns and Jam view moves to a Campaigns tab there). The `/api/inception/*` endpoints, the brainstorm agent and `/api/campaigns` are unchanged, and the wizard is still shown whenever sessions are off. + ## When to use it Use inception when you are standing up a brand-new hive at ACMM level 1 and diff --git a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go index 04bf41a306..2f0eb32430 100644 --- a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go +++ b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go @@ -10,14 +10,14 @@ import ( var dashboardLayoutIDs9062 = []string{ "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", - "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", + "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "sessions-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section", "faq-section", } var dashboardDefaultNavOrder9062 = []string{ "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", - "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", + "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "sessions-section", "acmm-eval-section", "debug-section", "audit-section", "faq-section", } @@ -26,7 +26,7 @@ var dashboardDefaultLayoutOrder9062 = append(append(append([]string{}, dashboard "faq-section", ) -const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` +const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','sessions-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` func jsArray9062(ids []string) string { var b strings.Builder @@ -165,7 +165,7 @@ func TestDashboardLayoutNormalizeFutureUnknownAndMissing(t *testing.T) { html := indexHTML(t) script := dashboardLayoutPreamble9062() + jsFunc(t, html, "dashboardLayoutAllIds") + "\n" + jsFunc(t, html, "dashboardLayoutNormalize") + ` const got = dashboardLayoutNormalize({v:99, main:['faq-section','beads-section','unknown','governor','faq-section']}); -const want = ` + jsArray9062([]string{"faq-section", "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section"}) + `; +const want = ` + jsArray9062([]string{"faq-section", "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "sessions-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section"}) + `; if (got.v !== 1) throw new Error('version not normalized: '+got.v); if (got.main.includes('beads-section')) throw new Error('legacy beads section survived normalization: '+JSON.stringify(got.main)); if (JSON.stringify(got.main) !== JSON.stringify(want)) throw new Error('normalized order '+JSON.stringify(got.main)); @@ -287,7 +287,7 @@ if (middle !== 'nous-section,logs-section,agents-section,agent-activity-section, if (new Set(got).size !== got.length) throw new Error('duplicate ids: '+JSON.stringify(got)); const moved = dashboardLayoutNormalize({v:1, main:['faq-section','cost-panel','overview-section']}).main; if (moved[0] !== 'faq-section') throw new Error('operator-moved FAQ not respected: '+JSON.stringify(moved)); -if (moved.length !== 22) throw new Error('missing sections not restored: '+moved.length); +if (moved.length !== 23) throw new Error('missing sections not restored: '+moved.length); ` cmd := exec.Command(node, "-e", script) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/src/pkg/dashboard/dashboard_section_summaries_test.go b/src/pkg/dashboard/dashboard_section_summaries_test.go index e1f1a7f317..deaad2ae18 100644 --- a/src/pkg/dashboard/dashboard_section_summaries_test.go +++ b/src/pkg/dashboard/dashboard_section_summaries_test.go @@ -39,7 +39,7 @@ func TestDashboardSectionCardsHaveCollapsedSummaries(t *testing.T) { "overview-section", "advisory-section", "hive-advice-section", "fleet-report-section", "acmm-reco-section", "lifecycle-section", "pr-throughput-section", "repos-section", "acmm-eval-section", "approvals-section", "audit-section", - "review-queue-section", "nous-section", "inception-section", "knowledge-section", "inbox-section", + "review-queue-section", "nous-section", "inception-section", "sessions-section", "knowledge-section", "inbox-section", "contributors-section", "debug-section", "logs-section", "agents-section", "faq-section", } for _, id := range refreshRequired { diff --git a/src/pkg/dashboard/sessions_ui.test.js b/src/pkg/dashboard/sessions_ui.test.js new file mode 100644 index 0000000000..e1cc802c5d --- /dev/null +++ b/src/pkg/dashboard/sessions_ui.test.js @@ -0,0 +1,108 @@ +// Sessions section (tabbed shared CLI sessions): tab grouping/selection state, +// role gating, input-bar limit checks and the layout feature gate that swaps +// Sessions in for Inception. Helpers are cut out of the generated dashboard so +// the tests run the shipped code. Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +const start = html.indexOf('const SESSIONS_API'); +const end = html.indexOf('// ── Storage conveniences'); +assert.ok(start >= 0 && end > start, 'sessions helper block not found in index.html'); +const names = ['sessionsRoleCanWrite', 'sessionsGroupSessions', 'sessionsExistingGroups', 'sessionsPickSelected', 'sessionsTabOrder', + 'sessionsNextTab', 'sessionsViewMode', 'sessionsCheckInput', 'sessionsCheckUpload', 'sessionsCanCreate', 'sessionsTerminalPath', + 'sessionsFormatBytes', 'sessionsIsEnabled']; +const h = new Function('window', html.slice(start, end) + '\nreturn {' + names.join(',') + '};')({}); + +const S = (id, group, status) => ({ id, name: id, group, status: status || 'running' }); + +test('sessions are grouped alphabetically with the ungrouped bucket last', () => { + const groups = h.sessionsGroupSessions([S('a', ''), S('b', 'zeta'), S('c', 'alpha'), S('d', 'alpha'), null, {}]); + assert.deepStrictEqual(groups.map(g => g.label), ['alpha', 'zeta', 'Ungrouped']); + assert.deepStrictEqual(groups[0].sessions.map(s => s.id), ['c', 'd']); + assert.deepStrictEqual(h.sessionsExistingGroups([S('a', ''), S('b', 'x'), S('c', 'x')]), ['x']); + assert.deepStrictEqual(h.sessionsGroupSessions(undefined), []); +}); + +test('selected tab falls back to the first session, then to Campaigns', () => { + const list = [S('a', ''), S('b', '')]; + assert.strictEqual(h.sessionsPickSelected(list, 'b'), 'b'); + assert.strictEqual(h.sessionsPickSelected(list, 'gone'), 'a'); + assert.strictEqual(h.sessionsPickSelected(list, 'campaigns'), 'campaigns'); + assert.strictEqual(h.sessionsPickSelected([], 'gone'), 'campaigns'); + assert.strictEqual(h.sessionsPickSelected(null, ''), 'campaigns'); +}); + +test('arrow keys walk the visible tabs and wrap; collapsed groups keep only the selected tab', () => { + const list = [S('a', 'g1'), S('b', 'g1'), S('c', 'g2')]; + const order = h.sessionsTabOrder(list, { g1: true }, 'b'); + assert.deepStrictEqual(order, ['b', 'c', 'campaigns']); + assert.strictEqual(h.sessionsNextTab(order, 'campaigns', 'ArrowRight'), 'b'); + assert.strictEqual(h.sessionsNextTab(order, 'b', 'ArrowLeft'), 'campaigns'); + assert.strictEqual(h.sessionsNextTab(order, 'c', 'Home'), 'b'); + assert.strictEqual(h.sessionsNextTab(order, 'b', 'End'), 'campaigns'); + assert.strictEqual(h.sessionsNextTab(order, 'b', 'x'), ''); + assert.strictEqual(h.sessionsNextTab([], 'b', 'ArrowRight'), ''); +}); + +test('only owner and read-write get a terminal; read sees scrollback; ended sessions never attach', () => { + const run = S('a', ''); + const ended = S('a', '', 'ended'); + for (const role of ['owner', 'read-write']) assert.strictEqual(h.sessionsViewMode(role, run), 'terminal'); + for (const role of ['read', 'merger', '', undefined]) assert.strictEqual(h.sessionsViewMode(role, run), 'scrollback'); + assert.strictEqual(h.sessionsViewMode('owner', ended), 'ended'); + assert.strictEqual(h.sessionsViewMode('owner', null), 'none'); + assert.strictEqual(h.sessionsRoleCanWrite('read'), false); +}); + +test('input bar rejects empty and oversize text using the server limits', () => { + const limits = { max_input_bytes: 10, max_upload_bytes: 100 }; + assert.strictEqual(h.sessionsCheckInput(' ', limits).ok, false); + assert.strictEqual(h.sessionsCheckInput('0123456789', limits).ok, true); + const over = h.sessionsCheckInput('01234567890', limits); + assert.strictEqual(over.ok, false); + assert.match(over.error, /limited to 10 bytes/); + assert.strictEqual(h.sessionsCheckInput('é'.repeat(6), limits).ok, false, 'limit counts bytes, not characters'); + assert.strictEqual(h.sessionsCheckInput('x'.repeat(1000), {}).ok, true, 'no limit advertised means no client cap'); +}); + +test('uploads are checked against max_upload_bytes before any request', () => { + const limits = { max_upload_bytes: 2048 }; + assert.strictEqual(h.sessionsCheckUpload(null, limits).ok, false); + assert.strictEqual(h.sessionsCheckUpload({ name: 'a', size: 0 }, limits).ok, false); + assert.strictEqual(h.sessionsCheckUpload({ name: 'a', size: 2048 }, limits).ok, true); + const big = h.sessionsCheckUpload({ name: 'big.bin', size: 3 * 1024 * 1024 }, limits); + assert.strictEqual(big.ok, false); + assert.match(big.error, /limited to 2 KiB/); +}); + +test('New session is gated by role, vendor availability and the running-session cap', () => { + const data = { vendors: [{ id: 'claude', available: true }, { id: 'codex', available: false }], sessions: [S('a', '')], limits: { max_sessions: 2 } }; + assert.strictEqual(h.sessionsCanCreate('owner', data).ok, true); + assert.strictEqual(h.sessionsCanCreate('read', data).ok, false); + assert.strictEqual(h.sessionsCanCreate('owner', { vendors: [{ id: 'x', available: false }], sessions: [], limits: {} }).ok, false); + const full = Object.assign({}, data, { sessions: [S('a', ''), S('b', ''), S('c', '', 'ended')] }); + assert.strictEqual(h.sessionsCanCreate('read-write', full).ok, false); + assert.match(h.sessionsCanCreate('read-write', full).reason, /limit of 2/); +}); + +test('terminal iframe path attaches hive-sess-<id> with the handoff code', () => { + assert.strictEqual(h.sessionsTerminalPath('abc', 'c0de'), '/terminal/?arg=hive-sess-abc&code=c0de'); + assert.strictEqual(h.sessionsTerminalPath('abc', ''), '/terminal/?arg=hive-sess-abc'); +}); + +test('layout gate shows Sessions and hides Inception only when sessions are enabled', () => { + const m = /function sessionsFeatureEnabled\(\)\{[^\n]*\n\s*function dashboardFeatureSectionHidden\(id\)\{[^\n]*\}/.exec(html); + assert.ok(m, 'layout gate not found'); + const gate = enabled => new Function('window', 'dashboardAliasSectionId', 'experimentLabEnabled', + m[0] + '\nreturn dashboardFeatureSectionHidden;')({ _sessionsEnabled: enabled }, id => id, () => true); + const off = gate(false); + assert.strictEqual(off('sessions-section'), true); + assert.strictEqual(off('inception-section'), false); + const on = gate(true); + assert.strictEqual(on('sessions-section'), false); + assert.strictEqual(on('inception-section'), true); + assert.strictEqual(on('nous-section'), false); +}); diff --git a/src/pkg/dashboard/static-src/css/sessions.css b/src/pkg/dashboard/static-src/css/sessions.css new file mode 100644 index 0000000000..ba64250fdd --- /dev/null +++ b/src/pkg/dashboard/static-src/css/sessions.css @@ -0,0 +1,50 @@ + /* Sessions section: tabbed, shared CLI terminals. Tokens only. */ + #sessions-section[hidden], + #inception-section[hidden], + .oc-nav-item[data-section="sessions-section"][hidden], + .oc-nav-item[data-section="inception-section"][hidden] { display: none !important; } + .sessions-root { display: flex; flex-direction: column; gap: var(--sp-4); } + .sessions-toolbar { display: flex; align-items: flex-start; gap: var(--sp-4); flex-wrap: wrap; } + .sessions-tabs { display: flex; flex-wrap: wrap; gap: var(--sp-4); flex: 1 1 auto; min-width: 0; } + .sessions-group { display: flex; flex-direction: column; gap: var(--sp-2); } + .sessions-group-toggle { background: none; border: 0; color: var(--muted); font: inherit; font-size: var(--fs-xs); text-align: left; cursor: pointer; padding: var(--sp-1) var(--sp-2); } + .sessions-group-toggle:hover { color: var(--fg); } + .sessions-group-chevron { margin-right: var(--sp-2); font-size: var(--fs-2xs); } + .sessions-group-count { opacity: 0.7; } + .sessions-group-tabs { display: flex; flex-wrap: wrap; gap: var(--sp-2); } + .sessions-tab { display: inline-flex; align-items: center; gap: var(--sp-3); background: var(--card-bg); color: var(--fg); border: 1px solid var(--border); border-radius: var(--r-sm); padding: var(--sp-3) var(--sp-4); font: inherit; font-size: var(--fs-sm); cursor: pointer; } + .sessions-tab:hover { border-color: var(--accent); } + .sessions-tab.active { border-color: var(--accent); background: color-mix(in srgb, var(--accent) 14%, transparent); font-weight: var(--fw-semibold); } + .sessions-tab.ended { opacity: 0.7; } + .sessions-tab:focus-visible, .sessions-group-toggle:focus-visible { outline: var(--focus-ring-width) solid var(--accent); outline-offset: var(--sp-1); } + .sessions-tab-dot { width: var(--status-dot-size); height: var(--status-dot-size); border-radius: var(--r-pill); background: var(--muted); } + .sessions-tab-dot[data-status="running"] { background: var(--green); } + .sessions-tab-vendor { color: var(--muted); font-size: var(--fs-xs); } + .sessions-attached { border: 1px solid var(--border); border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); color: var(--muted); font-size: var(--fs-xs); cursor: help; } + .sessions-new { border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-5); display: grid; gap: var(--sp-4); background: var(--card-bg); } + .sessions-new[hidden], .sessions-pane[hidden], #sessions-empty[hidden] { display: none; } + .sessions-form-row { display: grid; gap: var(--sp-2); } + .sessions-form-row label { font-size: var(--fs-xs); color: var(--muted); } + .sessions-form-actions, .sessions-input-actions { display: flex; flex-wrap: wrap; gap: var(--sp-3); align-items: center; } + .sessions-note { color: var(--muted); font-size: var(--fs-sm); } + .sessions-pane { display: flex; flex-direction: column; gap: var(--sp-4); } + .sessions-pane-head { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-4); } + .sessions-pane-title { font-weight: var(--fw-bold); } + .sessions-pane-meta { color: var(--muted); font-size: var(--fs-sm); flex: 1 1 auto; } + .sessions-menu { position: relative; } + .sessions-menu > summary { list-style: none; cursor: pointer; } + .sessions-menu-items { position: absolute; right: 0; z-index: var(--z-sticky); display: flex; flex-direction: column; min-width: 12em; background: var(--panel); border: 1px solid var(--border); border-radius: var(--r); box-shadow: var(--shadow-raised); padding: var(--sp-2); } + .sessions-menu-items button { background: none; border: 0; color: var(--fg); font: inherit; font-size: var(--fs-sm); text-align: left; padding: var(--sp-3) var(--sp-4); border-radius: var(--r-sm); cursor: pointer; } + .sessions-menu-items button:hover { background: color-mix(in srgb, var(--border) 30%, transparent); } + .sessions-menu-items button.danger { color: var(--red); } + .sessions-terminal { height: 65vh; min-height: 20em; border: 1px solid var(--border); border-radius: var(--r); overflow: hidden; background: var(--terminal-bg); } + .sessions-frame { width: 100%; height: 100%; border: 0; display: block; } + .sessions-log { height: 65vh; min-height: 20em; overflow: auto; margin: 0; padding: var(--sp-4); background: var(--terminal-bg); color: var(--terminal-fg, var(--fg)); border: 1px solid var(--border); border-radius: var(--r); font-family: var(--font-mono); font-size: var(--fs-xs); white-space: pre-wrap; overflow-wrap: anywhere; } + .sessions-input { display: grid; gap: var(--sp-3); } + .sessions-input-label { font-size: var(--fs-xs); color: var(--muted); } + .sessions-input-text { width: 100%; box-sizing: border-box; resize: vertical; background: var(--bg); color: var(--fg); border: 1px solid var(--border); border-radius: var(--r-sm); padding: var(--sp-4); font-family: var(--font-mono); font-size: var(--fs-sm); } + .sessions-attach { cursor: pointer; } + .sessions-file { position: absolute; width: var(--line-width); height: var(--line-width); opacity: 0; overflow: hidden; } + .sessions-attach:focus-within { outline: var(--focus-ring-width) solid var(--accent); } + .sessions-input-count { margin-left: auto; color: var(--muted); font-size: var(--fs-xs); } + .sessions-input-count.over { color: var(--red); } diff --git a/src/pkg/dashboard/static-src/html/faq.html b/src/pkg/dashboard/static-src/html/faq.html index 41ace5bf37..7b4654b364 100644 --- a/src/pkg/dashboard/static-src/html/faq.html +++ b/src/pkg/dashboard/static-src/html/faq.html @@ -132,6 +132,11 @@ <h3 class="faq-group-title">Runs, inception and the Experiment Lab</h3> <p>Inception is the L1 human-and-brainstorm workflow for turning an idea into facts and a scaffold. Spek is the optional runner for staged speks and plans once you are using long-running runs. Both keep authority in Hive: Spek status can advance a lease, but implement still lands through the existing PR flow. See <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/inception.md" target="_blank" rel="noopener noreferrer">Inception</a> and <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/spektacular.md" target="_blank" rel="noopener noreferrer">Spek</a>.</p> </details> + <details class="faq-q" id="faq-sessions"> + <summary>What are Sessions, and where did Inception go?</summary> + <p>When sessions are enabled on your hive, the dashboard shows a <strong>Sessions</strong> section instead of Inception: named terminal tabs where a command-line AI agent runs inside Hive, shared with everyone on the hive. People with the <code>read</code> role see a read-only scrollback. Typing in a terminal is shared and not attributed to individual people. Campaigns and Jam moved to a Campaigns tab. See <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/dashboard-sections/workflow-and-agents.md#sessions" target="_blank" rel="noopener noreferrer">Sessions</a>.</p> + </details> + <details class="faq-q" id="faq-external-engines"> <summary>What are external engines like Flue and OMP allowed to do?</summary> <p>External engines are report-only admission surfaces, not extra credentials: they receive bounded context and return receipts that Hive verifies, and they never receive target-repo write credentials or dashboard tokens. Flue and OMP are independent, default-off bindings (<code>runs.external.flue.enabled</code>, <code>runs.external.omp.enabled</code>), and dispatch additionally requires the relay to advertise <code>ext-exec/flue</code> or <code>ext-exec/omp</code>. See <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/design/external-workflow-admission.md" target="_blank" rel="noopener noreferrer">external workflow admission</a> and <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/contributor-relay.md" target="_blank" rel="noopener noreferrer">contributor relay capability tokens</a>.</p> diff --git a/src/pkg/dashboard/static-src/html/nous-inception-knowledge-contributors.html b/src/pkg/dashboard/static-src/html/nous-inception-knowledge-contributors.html index 2035629e91..a78ca013ca 100644 --- a/src/pkg/dashboard/static-src/html/nous-inception-knowledge-contributors.html +++ b/src/pkg/dashboard/static-src/html/nous-inception-knowledge-contributors.html @@ -13,6 +13,13 @@ <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" </div> </div> + <div class="mt-6 mb-8" id="sessions-section" data-dashboard-section="sessions-section" hidden> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="sessions-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Sessions section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 💻 Sessions</h2> + <div class="section-body"> + <div id="sessions-panel"></div> + </div> + </div> + <div class="mt-6 mb-8" id="knowledge-section" data-dashboard-section="knowledge-section"> <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="knowledge-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Knowledge section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📚 Knowledge <span id="kb-status-badge" style="font-size:0.7rem;padding:var(--sp-1) var(--sp-4);border-radius:var(--r-pill);margin-left:var(--sp-4)"></span><span id="public-knowledge-header-control" data-public-knowledge-control="header" data-stop="1"></span></h2> <div class="section-body"> diff --git a/src/pkg/dashboard/static-src/html/sidebar.html b/src/pkg/dashboard/static-src/html/sidebar.html index 9ea8dbb310..555b3fd7f5 100644 --- a/src/pkg/dashboard/static-src/html/sidebar.html +++ b/src/pkg/dashboard/static-src/html/sidebar.html @@ -41,6 +41,7 @@ <a class="oc-nav-item" data-section="advisory-section" data-action="ocNavigate" data-arg0="advisory-section"><span class="oc-nav-emoji">📣</span><span class="oc-nav-text">Advisory</span></a> <a class="oc-nav-item" data-section="acmm-eval-section" data-action="ocNavigate" data-arg0="acmm-eval-section"><span class="oc-nav-emoji">🎯</span><span class="oc-nav-text">ACMM Eval</span></a> <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> + <a class="oc-nav-item" data-section="sessions-section" data-action="ocNavigate" data-arg0="sessions-section" hidden><span class="oc-nav-emoji">💻</span><span class="oc-nav-text">Sessions</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> diff --git a/src/pkg/dashboard/static-src/js/agent-detail.js b/src/pkg/dashboard/static-src/js/agent-detail.js index 2a82dfd3d5..2ca957dc55 100644 --- a/src/pkg/dashboard/static-src/js/agent-detail.js +++ b/src/pkg/dashboard/static-src/js/agent-detail.js @@ -238,6 +238,8 @@ } function ocNavigate(section, skipScroll) { section = dashboardAliasSectionId(section); + // Sessions replaces Inception when enabled: old links and the welcome checklist land there. + if (section === 'inception-section' && window._sessionsEnabled === true) section = 'sessions-section'; if (dashboardFeatureSectionHidden(section)) return; if (dashboardSectionHidden(section)) dashboardSetSectionHidden(section,false); _ocSelectedAgent = null; diff --git a/src/pkg/dashboard/static-src/js/auth-headers.js b/src/pkg/dashboard/static-src/js/auth-headers.js new file mode 100644 index 0000000000..0af663652c --- /dev/null +++ b/src/pkg/dashboard/static-src/js/auth-headers.js @@ -0,0 +1,9 @@ + // ── Shared dashboard auth headers ── + // Bearer-token header helper used across the dashboard. It lived in + // inception.js; the name is kept so existing callers do not change. + function _inceptionAuthHeaders() { + var h = {'Content-Type': 'application/json'}; + var t = localStorage.getItem('hive-token'); + if (t) h['Authorization'] = 'Bearer ' + t; + return h; + } diff --git a/src/pkg/dashboard/static-src/js/campaigns.js b/src/pkg/dashboard/static-src/js/campaigns.js new file mode 100644 index 0000000000..deeabc4dfb --- /dev/null +++ b/src/pkg/dashboard/static-src/js/campaigns.js @@ -0,0 +1,390 @@ + // ── Campaigns + Jam (Spek / inception campaign archive) ── + // Moved verbatim out of inception.js so the Campaigns and Jam UI survives + // the Inception wizard being replaced by the Sessions section. Plain + // top-level declarations, shared with inception.js through the single + // concatenated dashboard script; keep this file adjacent to it in manifest.txt. + let _campaigns = []; + + async function loadCampaigns() { + try { + const params = new URLSearchParams(); + ['search', 'repo', 'stage', 'status', 'owner'].forEach(function(name) { + const el = document.getElementById('campaign-filter-' + name); + if (el && el.value.trim()) params.set(name, el.value.trim()); + }); + const qs = params.toString(); + const r = await fetch('/api/campaigns' + (qs ? '?' + qs : ''), {headers: _inceptionAuthHeaders()}); + const data = await r.json(); + if (data.ok) { + _campaigns = data.campaigns || []; + renderCampaignsList(); + } + } catch (e) { console.error('campaign load failed', e); } + } + + function renderCampaignsPanel() { + return ` + <div class="campaigns-card"> + <div class="campaigns-head"> + <div><div class="campaigns-title">Campaigns</div><div class="campaigns-subtitle">Resumable Inception and Spek sessions keyed by stable spec ID.</div></div> + <button class="hv-btn btn-secondary btn-sm" data-action="loadCampaigns">Refresh</button> + </div> + <div class="campaigns-filters"> + <input id="campaign-filter-search" class="campaigns-input" placeholder="Search campaigns" /> + <input id="campaign-filter-repo" class="campaigns-input" placeholder="Repo" /> + <input id="campaign-filter-stage" class="campaigns-input" placeholder="Stage" /> + <input id="campaign-filter-status" class="campaigns-input" placeholder="Status" /> + <input id="campaign-filter-owner" class="campaigns-input" placeholder="Owner" /> + <button class="hv-btn btn-secondary btn-sm" data-action="loadCampaigns">Apply</button> + </div> + <div id="campaigns-list" class="campaigns-list"><div class="campaign-empty">Loading campaigns…</div></div> + <div id="campaign-jam-panel" class="campaign-jam"></div> + </div>`; + } + + // Revise chains (#12698): every Revise mints a linked `<root>-rev-N` + // archive with the same title, so the API marks the older ones with + // `superseded_by`. Show one row per chain (the newest revision) and fold + // the earlier ones into a history disclosure under it. + function campaignRevisionChains(campaigns) { + const byId = {}; + (campaigns || []).forEach(function(c) { byId[c.id] = c; }); + const latest = (campaigns || []).filter(function(c) { return !c.superseded_by || !byId[c.superseded_by]; }); + return latest.map(function(head) { + const earlier = []; + const seen = {}; + let cur = head; + while (cur && cur.revision_of && byId[cur.revision_of] && !seen[cur.revision_of]) { + seen[cur.revision_of] = true; + cur = byId[cur.revision_of]; + earlier.push(cur); + } + return { head: head, history: earlier }; + }); + } + + function campaignDeleteButton(c) { + return nousExpIconButton('deleteCampaign', escapeHtml(c.id), '', 'Delete campaign', '🗑', 'nous-exp-icon-delete campaign-delete'); + } + + function renderCampaignRow(c, history) { + const repo = (c.repos || []).join(', ') || 'no repo'; + const activity = c.last_activity || 'no activity'; + const engine = c.engine || 'unknown engine'; + const rc = c.recheck || {}; + const recheckText = rc.enabled ? `Rechecks every ${escapeHtml(rc.interval || 'default')} · next ${escapeHtml(rc.next_at ? relativeAge(rc.next_at) : 'not scheduled')} · last delta ${Number(rc.last_delta_count || 0)}` : 'Rechecks off'; + const drift = c.drift || {}; + const external = Array.isArray(drift.external) ? drift.external : []; + const failed = Array.isArray(rc.sources_failed) ? rc.sources_failed : (Array.isArray(drift.sources_failed) ? drift.sources_failed : []); + const recheckExternal = external.length ? `<details class="campaign-row-meta"><summary>${Number(drift.external_count || external.length)} external signals</summary><div class="campaign-jam-list">${external.map(ev => `<div><strong>${escapeHtml(ev.source || '')}</strong> · ${escapeHtml(ev.kind || '')}: ${ev.url ? `<a href="${escapeHtml(ev.url)}" target="_blank" rel="noopener noreferrer">${escapeHtml(ev.title || ev.url)}</a>` : escapeHtml(ev.title || '')}${ev.summary ? `<br>${escapeHtml(ev.summary)}` : ''}</div>`).join('')}${failed.length ? `<div>Failed sources: ${failed.map(f => `${escapeHtml(f.name || '')} (${escapeHtml(f.reason || '')})`).join(', ')}</div>` : ''}</div></details>` : (Number(rc.external_count || 0) ? `<div class="campaign-row-meta"><span>${Number(rc.external_count || 0)} external signals</span></div>` : ''); + const recheckDisabled = rc.in_flight ? ' disabled' : ''; + const revisionPill = c.revision ? `<span class="campaign-pill campaign-revision-pill" title="Revision ${Number(c.revision)}${c.revision_of ? ' of ' + escapeHtml(c.revision_of) : ''}">rev ${Number(c.revision)}</span>` : ''; + const hist = Array.isArray(history) ? history : []; + const historyBlock = hist.length ? `<details class="campaign-row-meta campaign-history"><summary>${hist.length} earlier ${hist.length === 1 ? 'revision' : 'revisions'}</summary><div class="campaign-history-list">${hist.map(function(h) { + return `<div class="campaign-history-row" data-campaign-id="${escapeHtml(h.id)}"><span class="campaign-pill">${h.revision ? 'rev ' + Number(h.revision) : 'original'}</span><span>${escapeHtml(h.id)}</span><span>${escapeHtml(h.status || 'parked')}</span><span>${escapeHtml(h.last_activity || '')}</span><span class="campaign-row-actions">${nousExpIconButton('resumeCampaign', escapeHtml(h.id), '', 'Continue this revision', '▶', 'nous-exp-icon-resume')}${campaignDeleteButton(h)}</span></div>`; + }).join('')}</div></details>` : ''; + return `<div class="campaign-row" data-campaign-id="${escapeHtml(c.id)}"> + <div> + <div class="campaign-row-title">${escapeHtml(c.title || c.id)}</div> + <div class="campaign-row-meta"> + <span class="campaign-pill">${escapeHtml(c.status || 'parked')}</span> + ${revisionPill} + <span>${escapeHtml(c.current_stage || '')}${c.current_step ? ' / ' + escapeHtml(c.current_step) : ''}</span> + <span>${escapeHtml(repo)}</span> + <span>${escapeHtml(engine)}</span> + <span>${escapeHtml(activity)}</span> + </div> + <div class="campaign-row-meta"><span>${recheckText}</span></div> + ${recheckExternal} + ${historyBlock} + </div> + <div class="campaign-row-actions"> + <button class="hv-btn btn-secondary btn-sm" data-action="openCampaignJam" data-arg0="${escapeHtml(c.id)}">Jam</button> + <button class="hv-btn btn-secondary btn-sm" data-action="recheckCampaign" data-arg0="${escapeHtml(c.id)}"${recheckDisabled}>Recheck</button> + <button class="hv-btn btn-primary btn-sm" data-action="resumeCampaign" data-arg0="${escapeHtml(c.id)}">Continue</button> + <button class="hv-btn btn-secondary btn-sm" data-action="releaseCampaign" data-arg0="${escapeHtml(c.id)}">Release</button> + <button class="hv-btn btn-secondary btn-sm" data-action="reviseCampaign" data-arg0="${escapeHtml(c.id)}">Revise</button> + ${campaignDeleteButton(c)} + </div> + </div>`; + } + + function renderCampaignsList() { + const list = document.getElementById('campaigns-list'); + setSectionSummary('inception-section', inceptionSectionSummary()); + if (!list) return; + if (!_campaigns || _campaigns.length === 0) { + list.innerHTML = '<div class="campaign-empty">No campaigns yet. New Inception archives completed sessions here.</div>'; + return; + } + list.innerHTML = campaignRevisionChains(_campaigns).map(function(chain) { return renderCampaignRow(chain.head, chain.history); }).join(''); + } + + // Delete only removes the campaign record (saved inception state + wiki + // snapshot). KB facts, Jam threads and scaffolded repo files are kept. + // In-app hiveConfirm dialog; native browser dialogs are forbidden (#12698). + const CAMPAIGN_DELETE_CONFIRM_TITLE = 'Delete this campaign?'; + function campaignDeleteConfirmMessage(c) { + const label = (c && (c.title || c.id)) || 'this campaign'; + const rev = c && c.revision ? ` (revision ${Number(c.revision)})` : ''; + return `Removes the campaign record for "${label}"${rev}: its saved Inception state and the archived wiki snapshot. It disappears from this list and cannot be continued or revised.\n\nKept: knowledge-base facts, Jam threads, and anything already scaffolded or pushed to the repo. Other revisions in the same chain are not affected.`; + } + + async function deleteCampaign(id) { + if (!id) return; + const c = (_campaigns || []).find(function(x) { return x.id === id; }) || { id: id }; + const ok = await hiveConfirm({ title: CAMPAIGN_DELETE_CONFIRM_TITLE, message: campaignDeleteConfirmMessage(c), confirmLabel: 'Delete campaign', cancelLabel: 'Keep', danger: true }); + if (!ok) return; + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id), { method: 'DELETE', headers: _inceptionAuthHeaders() }); + const data = await r.json().catch(function() { return {}; }); + if (!r.ok || !data.ok) { showToast(data.error || 'Failed to delete campaign', 'error'); return; } + _campaigns = (_campaigns || []).filter(function(x) { return x.id !== id; }); + renderCampaignsList(); + showToast('Campaign deleted — knowledge-base facts and Jam threads kept', 'info'); + await loadCampaigns(); + } catch (e) { showToast('Error: ' + e.message, 'error'); } + } + + let _campaignJamSocket = null; + let _campaignJamLiveRevision = ''; + + async function openCampaignJam(id) { + if (!id) return; + const panel = document.getElementById('campaign-jam-panel'); + if (panel) panel.innerHTML = '<div class="campaign-empty">Loading Jam…</div>'; + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/jam', {headers: _inceptionAuthHeaders()}); + const data = await r.json(); + if (!data.ok) { throw new Error(data.error || 'Failed to load Jam'); } + renderCampaignJam(id, data.jam || {}); + connectCampaignJamLive(id); + } catch (e) { + if (panel) panel.innerHTML = '<div class="campaign-empty">Jam failed: ' + escapeHtml(e.message) + '</div>'; + } + } + + function renderCampaignJam(id, jam) { + const panel = document.getElementById('campaign-jam-panel'); + if (!panel) return; + const threads = jam.threads || []; + const suggestions = jam.suggestions || []; + const polls = jam.polls || []; + const revisions = jam.revisions || []; + const projectSync = jam.project_sync || {}; + _campaignJamLiveRevision = jam.spec_revision_id || ''; + panel.innerHTML = ` + <div class="campaigns-head"> + <div><div class="campaigns-title">Jam: ${escapeHtml(id)}</div><div class="campaigns-subtitle">Async section threads, suggestions, polls, decisions and attributed spec revisions.</div></div> + <button class="hv-btn btn-secondary btn-sm" data-action="openCampaignJam" data-arg0="${escapeHtml(id)}">Refresh Jam</button> + </div> + <div class="campaign-jam-box"> + <h4>Live session</h4> + <div id="campaign-jam-live-status" class="campaign-jam-live">Connecting…</div> + <div class="campaign-jam-form"> + <input id="jam-live-section" class="campaigns-input" placeholder="Section you are editing" /> + <textarea id="jam-live-content" class="campaigns-input" placeholder="Draft spec content for conflict-aware live edit">${escapeHtml(jam.spec_content || '')}</textarea> + <div class="campaign-jam-actions"> + <button class="hv-btn btn-secondary btn-sm" data-action="focusCampaignJamLive" data-arg0="${escapeHtml(id)}">Share focus</button> + <button class="hv-btn btn-primary btn-sm" data-action="submitCampaignJamLiveEdit" data-arg0="${escapeHtml(id)}">Submit live edit</button> + </div> + </div> + </div> + <div class="campaign-jam-grid"> + <div class="campaign-jam-box"> + <h4>Threads</h4> + <div class="campaign-jam-list">${threads.length ? threads.map(t => `<div><strong>${escapeHtml(t.section || 'section')}</strong>: ${escapeHtml((t.comments && t.comments[0] && t.comments[0].body) || t.title || '')}<div class="campaign-jam-actions"><button class="hv-btn btn-secondary btn-sm" data-action="inviteCampaignJamAgent" data-arg0="${escapeHtml(id)}" data-arg1="${escapeHtml(t.id)}">Invite Spektacular</button></div></div>`).join('') : 'No threads yet.'}</div> + <div class="campaign-jam-form"> + <input id="jam-thread-section" class="campaigns-input" placeholder="Spec section" /> + <textarea id="jam-thread-body" class="campaigns-input" placeholder="Comment"></textarea> + <button class="hv-btn btn-primary btn-sm" data-action="createCampaignJamThread" data-arg0="${escapeHtml(id)}">Add thread</button> + </div> + </div> + <div class="campaign-jam-box"> + <h4>Suggestions</h4> + <div class="campaign-jam-list">${suggestions.length ? suggestions.map(s => `<div><strong>${escapeHtml(s.section || 'section')}</strong> · ${escapeHtml(s.status || 'open')}<br>${escapeHtml(s.proposed_text || '')}<div class="campaign-jam-actions">${s.status === 'open' ? `<button class="hv-btn btn-secondary btn-sm" data-action="resolveCampaignJamSuggestion" data-arg0="${escapeHtml(id)}" data-arg1="${escapeHtml(s.id)}" data-arg2="accept">Accept</button><button class="hv-btn btn-secondary btn-sm" data-action="resolveCampaignJamSuggestion" data-arg0="${escapeHtml(id)}" data-arg1="${escapeHtml(s.id)}" data-arg2="reject">Reject</button>` : ''}</div></div>`).join('') : 'No suggestions yet.'}</div> + <div class="campaign-jam-form"> + <input id="jam-suggestion-section" class="campaigns-input" placeholder="Spec section" /> + <textarea id="jam-suggestion-text" class="campaigns-input" placeholder="Suggested text"></textarea> + <button class="hv-btn btn-primary btn-sm" data-action="createCampaignJamSuggestion" data-arg0="${escapeHtml(id)}">Suggest text</button> + </div> + </div> + <div class="campaign-jam-box"> + <h4>Polls & decisions</h4> + <div class="campaign-jam-list">${polls.length ? polls.map(p => `<div><strong>${escapeHtml(p.question || '')}</strong><br>${(p.options || []).map(o => `${escapeHtml(o.label)}: ${o.count || 0}`).join(' · ')}${p.decision ? `<br>Decision: ${escapeHtml(p.decision.outcome)} — ${escapeHtml(p.decision.rationale)}` : ''}</div>`).join('') : 'No polls yet.'}</div> + <div class="campaign-jam-form"> + <input id="jam-poll-section" class="campaigns-input" placeholder="Spec section" /> + <input id="jam-poll-question" class="campaigns-input" placeholder="Question" /> + <input id="jam-poll-options" class="campaigns-input" placeholder="Options separated by commas" /> + <button class="hv-btn btn-primary btn-sm" data-action="createCampaignJamPoll" data-arg0="${escapeHtml(id)}">Create poll</button> + </div> + </div> + </div> + <div class="campaign-jam-box"> + <h4>Revision history</h4> + <div class="campaign-jam-list">${revisions.length ? revisions.map(r => `<div><strong>${escapeHtml(r.id)}</strong> by ${escapeHtml((r.author && r.author.name) || 'unknown')}${r.model ? ' · ' + escapeHtml(r.model) : ''}<br>${escapeHtml(r.reason || 'revision')}</div>`).join('') : 'No spec revisions yet.'}</div> + </div> + <div class="campaign-jam-box"> + <h4>GitHub Projects sync</h4> + <div class="campaign-jam-list"> + <div>Status: ${escapeHtml(projectSync.last_status || (projectSync.enabled ? 'enabled' : 'disabled'))}</div> + ${projectSync.project_url ? `<div>Project: ${escapeHtml(projectSync.project_url)}</div>` : ''} + ${projectSync.last_error ? `<div>Last error: ${escapeHtml(projectSync.last_error)}<br>${escapeHtml(projectSync.retry_advice || 'Retry after checking the project link and token.')}</div>` : ''} + </div> + <div class="campaign-jam-form"> + <input id="jam-project-url" class="campaigns-input" placeholder="GitHub Projects URL or ID" value="${escapeHtml(projectSync.project_url || projectSync.project_id || '')}" /> + <div class="campaign-jam-actions"> + <button class="hv-btn btn-secondary btn-sm" data-action="enableCampaignJamProjectSync" data-arg0="${escapeHtml(id)}">Enable sync</button> + <button class="hv-btn btn-primary btn-sm" data-action="syncCampaignJamProject" data-arg0="${escapeHtml(id)}">Sync now</button> + <button class="hv-btn btn-secondary btn-sm" data-action="disableCampaignJamProjectSync" data-arg0="${escapeHtml(id)}">Disable</button> + </div> + </div> + </div>`; + } + + function connectCampaignJamLive(id) { + if (_campaignJamSocket) { + const prev = _campaignJamSocket; + _campaignJamSocket = null; + prev.onmessage = null; + prev.onclose = null; + prev.close(); + } + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const url = protocol + '//' + window.location.host + '/api/campaigns/' + encodeURIComponent(id) + '/jam/ws'; + try { + const ws = new WebSocket(url); + _campaignJamSocket = ws; + ws.onmessage = function(event) { + const msg = JSON.parse(event.data || '{}'); + if (msg.jam && msg.jam.spec_revision_id) { _campaignJamLiveRevision = msg.jam.spec_revision_id; } + if (msg.type === 'snapshot' || msg.type === 'presence') { renderCampaignJamPresence(msg.presence || []); } + if (msg.type === 'edit_applied' && msg.jam) { renderCampaignJam(id, msg.jam); connectCampaignJamLive(id); } + if (msg.type === 'conflict') { showToast('Jam edit conflict: refresh or merge against revision ' + (msg.expected_revision_id || 'current'), 'error'); } + if (msg.type === 'error') { showToast('Jam live error: ' + (msg.error || 'unknown error'), 'error'); } + }; + ws.onclose = function() { renderCampaignJamPresence([], 'Disconnected; refresh Jam to reconnect.'); }; + } catch (e) { + renderCampaignJamPresence([], 'Live unavailable: ' + e.message); + } + } + + function renderCampaignJamPresence(presence, message) { + const el = document.getElementById('campaign-jam-live-status'); + if (!el) return; + if (message) { el.textContent = message; return; } + el.innerHTML = presence.length ? presence.map(function(p) { + const name = (p.actor && p.actor.name) || 'participant'; + return '<span class="campaign-jam-live-pill">' + escapeHtml(name) + (p.section ? ' · ' + escapeHtml(p.section) : '') + '</span>'; + }).join('') : 'No live participants yet.'; + } + + function focusCampaignJamLive() { + if (!_campaignJamSocket || _campaignJamSocket.readyState !== WebSocket.OPEN) return; + const section = document.getElementById('jam-live-section')?.value || ''; + _campaignJamSocket.send(JSON.stringify({ type: 'focus', section })); + } + + function submitCampaignJamLiveEdit() { + if (!_campaignJamSocket || _campaignJamSocket.readyState !== WebSocket.OPEN) { showToast('Jam live socket is not connected', 'error'); return; } + const content = document.getElementById('jam-live-content')?.value || ''; + _campaignJamSocket.send(JSON.stringify({ type: 'edit', base_revision_id: _campaignJamLiveRevision, content, reason: 'live co-edit' })); + } + + async function createCampaignJamThread(id) { + const section = document.getElementById('jam-thread-section')?.value || ''; + const body = document.getElementById('jam-thread-body')?.value || ''; + await postCampaignJam(id, 'threads', { section, body }); + } + + async function inviteCampaignJamAgent(id, thread_id) { + const prompt = await promptForJamAgent(); + if (prompt === null) return; + await postCampaignJam(id, 'agents', { thread_id, agent: 'spektacular', prompt }); + } + + function promptForJamAgent() { + return hivePrompt({title: 'Prompt for Jam agent', label: 'Prompt for the Jam agent reply and suggestion', defaultValue: 'Suggest spec text for this thread.'}); + } + + async function createCampaignJamSuggestion(id) { + const section = document.getElementById('jam-suggestion-section')?.value || ''; + const proposed_text = document.getElementById('jam-suggestion-text')?.value || ''; + await postCampaignJam(id, 'suggestions', { section, proposed_text }); + } + + async function resolveCampaignJamSuggestion(id, suggestion_id, action) { + await postCampaignJam(id, 'suggestions', { action, suggestion_id }); + } + + async function createCampaignJamPoll(id) { + const section = document.getElementById('jam-poll-section')?.value || ''; + const question = document.getElementById('jam-poll-question')?.value || ''; + const options = (document.getElementById('jam-poll-options')?.value || '').split(',').map(s => s.trim()).filter(Boolean); + await postCampaignJam(id, 'polls', { section, question, options }); + } + + async function enableCampaignJamProjectSync(id) { + const value = document.getElementById('jam-project-url')?.value || ''; + const body = value.indexOf('http') === 0 ? { action: 'enable', project_url: value } : { action: 'enable', project_id: value }; + await postCampaignJam(id, 'project-sync', body); + } + + async function syncCampaignJamProject(id) { + await postCampaignJam(id, 'project-sync', { action: 'sync' }); + } + + async function disableCampaignJamProjectSync(id) { + await postCampaignJam(id, 'project-sync', { action: 'disable' }); + } + + async function postCampaignJam(id, lane, body) { + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/jam/' + lane, { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify(body) }); + const data = await r.json(); + if (!data.ok) { showToast(data.error || 'Jam update failed', 'error'); return; } + renderCampaignJam(id, data.jam || {}); + } catch (e) { showToast('Jam error: ' + e.message, 'error'); } + } + + async function resumeCampaign(id) { + if (!id) return; + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/resume', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({ surface: 'dashboard' }) }); + const data = await r.json(); + if (!data.ok) { showToast(data.error || 'Failed to resume campaign', 'error'); return; } + if (data.state) { _inceptionState = data.state; renderInception(); } + if (data.resume_command) { await hiveAlert('Resume command:\n' + data.resume_command, 'Resume command'); } + await loadCampaigns(); + } catch (e) { showToast('Error: ' + e.message, 'error'); } + } + + async function releaseCampaign(id) { + if (!id) return; + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/release', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({}) }); + const data = await r.json(); + if (!data.ok) { showToast(data.error || 'Failed to release campaign', 'error'); return; } + await loadCampaigns(); + } catch (e) { showToast('Error: ' + e.message, 'error'); } + } + + async function reviseCampaign(id) { + if (!id) return; + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/revise', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({}) }); + const data = await r.json(); + if (!data.ok) { showToast(data.error || 'Failed to revise campaign', 'error'); return; } + await loadCampaigns(); + } catch (e) { showToast('Error: ' + e.message, 'error'); } + } + + async function recheckCampaign(id) { + if (!id) return; + try { + const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/recheck', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({}) }); + const data = await r.json(); + if (!data.ok) { showToast(data.error || 'Failed to recheck campaign', 'error'); return; } + await loadCampaigns(); + } catch (e) { showToast('Error: ' + e.message, 'error'); } + } diff --git a/src/pkg/dashboard/static-src/js/inception.js b/src/pkg/dashboard/static-src/js/inception.js index 0dcc8e94c8..49685f5362 100644 --- a/src/pkg/dashboard/static-src/js/inception.js +++ b/src/pkg/dashboard/static-src/js/inception.js @@ -1,6 +1,5 @@ // ── Inception section ── let _inceptionState = null; - let _campaigns = []; async function loadInceptionState() { @@ -13,23 +12,6 @@ } catch (e) { console.error('inception state load failed', e); } } - async function loadCampaigns() { - try { - const params = new URLSearchParams(); - ['search', 'repo', 'stage', 'status', 'owner'].forEach(function(name) { - const el = document.getElementById('campaign-filter-' + name); - if (el && el.value.trim()) params.set(name, el.value.trim()); - }); - const qs = params.toString(); - const r = await fetch('/api/campaigns' + (qs ? '?' + qs : ''), {headers: _inceptionAuthHeaders()}); - const data = await r.json(); - if (data.ok) { - _campaigns = data.campaigns || []; - renderCampaignsList(); - } - } catch (e) { console.error('campaign load failed', e); } - } - function renderInception() { refreshSectionCardShell('inception-section'); setSectionSummary('inception-section', inceptionSectionSummary()); @@ -82,373 +64,6 @@ if (!_inceptionPollTimer) startInceptionPoll(); } - function renderCampaignsPanel() { - return ` - <div class="campaigns-card"> - <div class="campaigns-head"> - <div><div class="campaigns-title">Campaigns</div><div class="campaigns-subtitle">Resumable Inception and Spek sessions keyed by stable spec ID.</div></div> - <button class="hv-btn btn-secondary btn-sm" data-action="loadCampaigns">Refresh</button> - </div> - <div class="campaigns-filters"> - <input id="campaign-filter-search" class="campaigns-input" placeholder="Search campaigns" /> - <input id="campaign-filter-repo" class="campaigns-input" placeholder="Repo" /> - <input id="campaign-filter-stage" class="campaigns-input" placeholder="Stage" /> - <input id="campaign-filter-status" class="campaigns-input" placeholder="Status" /> - <input id="campaign-filter-owner" class="campaigns-input" placeholder="Owner" /> - <button class="hv-btn btn-secondary btn-sm" data-action="loadCampaigns">Apply</button> - </div> - <div id="campaigns-list" class="campaigns-list"><div class="campaign-empty">Loading campaigns…</div></div> - <div id="campaign-jam-panel" class="campaign-jam"></div> - </div>`; - } - - // Revise chains (#12698): every Revise mints a linked `<root>-rev-N` - // archive with the same title, so the API marks the older ones with - // `superseded_by`. Show one row per chain (the newest revision) and fold - // the earlier ones into a history disclosure under it. - function campaignRevisionChains(campaigns) { - const byId = {}; - (campaigns || []).forEach(function(c) { byId[c.id] = c; }); - const latest = (campaigns || []).filter(function(c) { return !c.superseded_by || !byId[c.superseded_by]; }); - return latest.map(function(head) { - const earlier = []; - const seen = {}; - let cur = head; - while (cur && cur.revision_of && byId[cur.revision_of] && !seen[cur.revision_of]) { - seen[cur.revision_of] = true; - cur = byId[cur.revision_of]; - earlier.push(cur); - } - return { head: head, history: earlier }; - }); - } - - function campaignDeleteButton(c) { - return nousExpIconButton('deleteCampaign', escapeHtml(c.id), '', 'Delete campaign', '🗑', 'nous-exp-icon-delete campaign-delete'); - } - - function renderCampaignRow(c, history) { - const repo = (c.repos || []).join(', ') || 'no repo'; - const activity = c.last_activity || 'no activity'; - const engine = c.engine || 'unknown engine'; - const rc = c.recheck || {}; - const recheckText = rc.enabled ? `Rechecks every ${escapeHtml(rc.interval || 'default')} · next ${escapeHtml(rc.next_at ? relativeAge(rc.next_at) : 'not scheduled')} · last delta ${Number(rc.last_delta_count || 0)}` : 'Rechecks off'; - const drift = c.drift || {}; - const external = Array.isArray(drift.external) ? drift.external : []; - const failed = Array.isArray(rc.sources_failed) ? rc.sources_failed : (Array.isArray(drift.sources_failed) ? drift.sources_failed : []); - const recheckExternal = external.length ? `<details class="campaign-row-meta"><summary>${Number(drift.external_count || external.length)} external signals</summary><div class="campaign-jam-list">${external.map(ev => `<div><strong>${escapeHtml(ev.source || '')}</strong> · ${escapeHtml(ev.kind || '')}: ${ev.url ? `<a href="${escapeHtml(ev.url)}" target="_blank" rel="noopener noreferrer">${escapeHtml(ev.title || ev.url)}</a>` : escapeHtml(ev.title || '')}${ev.summary ? `<br>${escapeHtml(ev.summary)}` : ''}</div>`).join('')}${failed.length ? `<div>Failed sources: ${failed.map(f => `${escapeHtml(f.name || '')} (${escapeHtml(f.reason || '')})`).join(', ')}</div>` : ''}</div></details>` : (Number(rc.external_count || 0) ? `<div class="campaign-row-meta"><span>${Number(rc.external_count || 0)} external signals</span></div>` : ''); - const recheckDisabled = rc.in_flight ? ' disabled' : ''; - const revisionPill = c.revision ? `<span class="campaign-pill campaign-revision-pill" title="Revision ${Number(c.revision)}${c.revision_of ? ' of ' + escapeHtml(c.revision_of) : ''}">rev ${Number(c.revision)}</span>` : ''; - const hist = Array.isArray(history) ? history : []; - const historyBlock = hist.length ? `<details class="campaign-row-meta campaign-history"><summary>${hist.length} earlier ${hist.length === 1 ? 'revision' : 'revisions'}</summary><div class="campaign-history-list">${hist.map(function(h) { - return `<div class="campaign-history-row" data-campaign-id="${escapeHtml(h.id)}"><span class="campaign-pill">${h.revision ? 'rev ' + Number(h.revision) : 'original'}</span><span>${escapeHtml(h.id)}</span><span>${escapeHtml(h.status || 'parked')}</span><span>${escapeHtml(h.last_activity || '')}</span><span class="campaign-row-actions">${nousExpIconButton('resumeCampaign', escapeHtml(h.id), '', 'Continue this revision', '▶', 'nous-exp-icon-resume')}${campaignDeleteButton(h)}</span></div>`; - }).join('')}</div></details>` : ''; - return `<div class="campaign-row" data-campaign-id="${escapeHtml(c.id)}"> - <div> - <div class="campaign-row-title">${escapeHtml(c.title || c.id)}</div> - <div class="campaign-row-meta"> - <span class="campaign-pill">${escapeHtml(c.status || 'parked')}</span> - ${revisionPill} - <span>${escapeHtml(c.current_stage || '')}${c.current_step ? ' / ' + escapeHtml(c.current_step) : ''}</span> - <span>${escapeHtml(repo)}</span> - <span>${escapeHtml(engine)}</span> - <span>${escapeHtml(activity)}</span> - </div> - <div class="campaign-row-meta"><span>${recheckText}</span></div> - ${recheckExternal} - ${historyBlock} - </div> - <div class="campaign-row-actions"> - <button class="hv-btn btn-secondary btn-sm" data-action="openCampaignJam" data-arg0="${escapeHtml(c.id)}">Jam</button> - <button class="hv-btn btn-secondary btn-sm" data-action="recheckCampaign" data-arg0="${escapeHtml(c.id)}"${recheckDisabled}>Recheck</button> - <button class="hv-btn btn-primary btn-sm" data-action="resumeCampaign" data-arg0="${escapeHtml(c.id)}">Continue</button> - <button class="hv-btn btn-secondary btn-sm" data-action="releaseCampaign" data-arg0="${escapeHtml(c.id)}">Release</button> - <button class="hv-btn btn-secondary btn-sm" data-action="reviseCampaign" data-arg0="${escapeHtml(c.id)}">Revise</button> - ${campaignDeleteButton(c)} - </div> - </div>`; - } - - function renderCampaignsList() { - const list = document.getElementById('campaigns-list'); - setSectionSummary('inception-section', inceptionSectionSummary()); - if (!list) return; - if (!_campaigns || _campaigns.length === 0) { - list.innerHTML = '<div class="campaign-empty">No campaigns yet. New Inception archives completed sessions here.</div>'; - return; - } - list.innerHTML = campaignRevisionChains(_campaigns).map(function(chain) { return renderCampaignRow(chain.head, chain.history); }).join(''); - } - - // Delete only removes the campaign record (saved inception state + wiki - // snapshot). KB facts, Jam threads and scaffolded repo files are kept. - // In-app hiveConfirm dialog; native browser dialogs are forbidden (#12698). - const CAMPAIGN_DELETE_CONFIRM_TITLE = 'Delete this campaign?'; - function campaignDeleteConfirmMessage(c) { - const label = (c && (c.title || c.id)) || 'this campaign'; - const rev = c && c.revision ? ` (revision ${Number(c.revision)})` : ''; - return `Removes the campaign record for "${label}"${rev}: its saved Inception state and the archived wiki snapshot. It disappears from this list and cannot be continued or revised.\n\nKept: knowledge-base facts, Jam threads, and anything already scaffolded or pushed to the repo. Other revisions in the same chain are not affected.`; - } - - async function deleteCampaign(id) { - if (!id) return; - const c = (_campaigns || []).find(function(x) { return x.id === id; }) || { id: id }; - const ok = await hiveConfirm({ title: CAMPAIGN_DELETE_CONFIRM_TITLE, message: campaignDeleteConfirmMessage(c), confirmLabel: 'Delete campaign', cancelLabel: 'Keep', danger: true }); - if (!ok) return; - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id), { method: 'DELETE', headers: _inceptionAuthHeaders() }); - const data = await r.json().catch(function() { return {}; }); - if (!r.ok || !data.ok) { showToast(data.error || 'Failed to delete campaign', 'error'); return; } - _campaigns = (_campaigns || []).filter(function(x) { return x.id !== id; }); - renderCampaignsList(); - showToast('Campaign deleted — knowledge-base facts and Jam threads kept', 'info'); - await loadCampaigns(); - } catch (e) { showToast('Error: ' + e.message, 'error'); } - } - - let _campaignJamSocket = null; - let _campaignJamLiveRevision = ''; - - async function openCampaignJam(id) { - if (!id) return; - const panel = document.getElementById('campaign-jam-panel'); - if (panel) panel.innerHTML = '<div class="campaign-empty">Loading Jam…</div>'; - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/jam', {headers: _inceptionAuthHeaders()}); - const data = await r.json(); - if (!data.ok) { throw new Error(data.error || 'Failed to load Jam'); } - renderCampaignJam(id, data.jam || {}); - connectCampaignJamLive(id); - } catch (e) { - if (panel) panel.innerHTML = '<div class="campaign-empty">Jam failed: ' + escapeHtml(e.message) + '</div>'; - } - } - - function renderCampaignJam(id, jam) { - const panel = document.getElementById('campaign-jam-panel'); - if (!panel) return; - const threads = jam.threads || []; - const suggestions = jam.suggestions || []; - const polls = jam.polls || []; - const revisions = jam.revisions || []; - const projectSync = jam.project_sync || {}; - _campaignJamLiveRevision = jam.spec_revision_id || ''; - panel.innerHTML = ` - <div class="campaigns-head"> - <div><div class="campaigns-title">Jam: ${escapeHtml(id)}</div><div class="campaigns-subtitle">Async section threads, suggestions, polls, decisions and attributed spec revisions.</div></div> - <button class="hv-btn btn-secondary btn-sm" data-action="openCampaignJam" data-arg0="${escapeHtml(id)}">Refresh Jam</button> - </div> - <div class="campaign-jam-box"> - <h4>Live session</h4> - <div id="campaign-jam-live-status" class="campaign-jam-live">Connecting…</div> - <div class="campaign-jam-form"> - <input id="jam-live-section" class="campaigns-input" placeholder="Section you are editing" /> - <textarea id="jam-live-content" class="campaigns-input" placeholder="Draft spec content for conflict-aware live edit">${escapeHtml(jam.spec_content || '')}</textarea> - <div class="campaign-jam-actions"> - <button class="hv-btn btn-secondary btn-sm" data-action="focusCampaignJamLive" data-arg0="${escapeHtml(id)}">Share focus</button> - <button class="hv-btn btn-primary btn-sm" data-action="submitCampaignJamLiveEdit" data-arg0="${escapeHtml(id)}">Submit live edit</button> - </div> - </div> - </div> - <div class="campaign-jam-grid"> - <div class="campaign-jam-box"> - <h4>Threads</h4> - <div class="campaign-jam-list">${threads.length ? threads.map(t => `<div><strong>${escapeHtml(t.section || 'section')}</strong>: ${escapeHtml((t.comments && t.comments[0] && t.comments[0].body) || t.title || '')}<div class="campaign-jam-actions"><button class="hv-btn btn-secondary btn-sm" data-action="inviteCampaignJamAgent" data-arg0="${escapeHtml(id)}" data-arg1="${escapeHtml(t.id)}">Invite Spektacular</button></div></div>`).join('') : 'No threads yet.'}</div> - <div class="campaign-jam-form"> - <input id="jam-thread-section" class="campaigns-input" placeholder="Spec section" /> - <textarea id="jam-thread-body" class="campaigns-input" placeholder="Comment"></textarea> - <button class="hv-btn btn-primary btn-sm" data-action="createCampaignJamThread" data-arg0="${escapeHtml(id)}">Add thread</button> - </div> - </div> - <div class="campaign-jam-box"> - <h4>Suggestions</h4> - <div class="campaign-jam-list">${suggestions.length ? suggestions.map(s => `<div><strong>${escapeHtml(s.section || 'section')}</strong> · ${escapeHtml(s.status || 'open')}<br>${escapeHtml(s.proposed_text || '')}<div class="campaign-jam-actions">${s.status === 'open' ? `<button class="hv-btn btn-secondary btn-sm" data-action="resolveCampaignJamSuggestion" data-arg0="${escapeHtml(id)}" data-arg1="${escapeHtml(s.id)}" data-arg2="accept">Accept</button><button class="hv-btn btn-secondary btn-sm" data-action="resolveCampaignJamSuggestion" data-arg0="${escapeHtml(id)}" data-arg1="${escapeHtml(s.id)}" data-arg2="reject">Reject</button>` : ''}</div></div>`).join('') : 'No suggestions yet.'}</div> - <div class="campaign-jam-form"> - <input id="jam-suggestion-section" class="campaigns-input" placeholder="Spec section" /> - <textarea id="jam-suggestion-text" class="campaigns-input" placeholder="Suggested text"></textarea> - <button class="hv-btn btn-primary btn-sm" data-action="createCampaignJamSuggestion" data-arg0="${escapeHtml(id)}">Suggest text</button> - </div> - </div> - <div class="campaign-jam-box"> - <h4>Polls & decisions</h4> - <div class="campaign-jam-list">${polls.length ? polls.map(p => `<div><strong>${escapeHtml(p.question || '')}</strong><br>${(p.options || []).map(o => `${escapeHtml(o.label)}: ${o.count || 0}`).join(' · ')}${p.decision ? `<br>Decision: ${escapeHtml(p.decision.outcome)} — ${escapeHtml(p.decision.rationale)}` : ''}</div>`).join('') : 'No polls yet.'}</div> - <div class="campaign-jam-form"> - <input id="jam-poll-section" class="campaigns-input" placeholder="Spec section" /> - <input id="jam-poll-question" class="campaigns-input" placeholder="Question" /> - <input id="jam-poll-options" class="campaigns-input" placeholder="Options separated by commas" /> - <button class="hv-btn btn-primary btn-sm" data-action="createCampaignJamPoll" data-arg0="${escapeHtml(id)}">Create poll</button> - </div> - </div> - </div> - <div class="campaign-jam-box"> - <h4>Revision history</h4> - <div class="campaign-jam-list">${revisions.length ? revisions.map(r => `<div><strong>${escapeHtml(r.id)}</strong> by ${escapeHtml((r.author && r.author.name) || 'unknown')}${r.model ? ' · ' + escapeHtml(r.model) : ''}<br>${escapeHtml(r.reason || 'revision')}</div>`).join('') : 'No spec revisions yet.'}</div> - </div> - <div class="campaign-jam-box"> - <h4>GitHub Projects sync</h4> - <div class="campaign-jam-list"> - <div>Status: ${escapeHtml(projectSync.last_status || (projectSync.enabled ? 'enabled' : 'disabled'))}</div> - ${projectSync.project_url ? `<div>Project: ${escapeHtml(projectSync.project_url)}</div>` : ''} - ${projectSync.last_error ? `<div>Last error: ${escapeHtml(projectSync.last_error)}<br>${escapeHtml(projectSync.retry_advice || 'Retry after checking the project link and token.')}</div>` : ''} - </div> - <div class="campaign-jam-form"> - <input id="jam-project-url" class="campaigns-input" placeholder="GitHub Projects URL or ID" value="${escapeHtml(projectSync.project_url || projectSync.project_id || '')}" /> - <div class="campaign-jam-actions"> - <button class="hv-btn btn-secondary btn-sm" data-action="enableCampaignJamProjectSync" data-arg0="${escapeHtml(id)}">Enable sync</button> - <button class="hv-btn btn-primary btn-sm" data-action="syncCampaignJamProject" data-arg0="${escapeHtml(id)}">Sync now</button> - <button class="hv-btn btn-secondary btn-sm" data-action="disableCampaignJamProjectSync" data-arg0="${escapeHtml(id)}">Disable</button> - </div> - </div> - </div>`; - } - - function connectCampaignJamLive(id) { - if (_campaignJamSocket) { - const prev = _campaignJamSocket; - _campaignJamSocket = null; - prev.onmessage = null; - prev.onclose = null; - prev.close(); - } - const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; - const url = protocol + '//' + window.location.host + '/api/campaigns/' + encodeURIComponent(id) + '/jam/ws'; - try { - const ws = new WebSocket(url); - _campaignJamSocket = ws; - ws.onmessage = function(event) { - const msg = JSON.parse(event.data || '{}'); - if (msg.jam && msg.jam.spec_revision_id) { _campaignJamLiveRevision = msg.jam.spec_revision_id; } - if (msg.type === 'snapshot' || msg.type === 'presence') { renderCampaignJamPresence(msg.presence || []); } - if (msg.type === 'edit_applied' && msg.jam) { renderCampaignJam(id, msg.jam); connectCampaignJamLive(id); } - if (msg.type === 'conflict') { showToast('Jam edit conflict: refresh or merge against revision ' + (msg.expected_revision_id || 'current'), 'error'); } - if (msg.type === 'error') { showToast('Jam live error: ' + (msg.error || 'unknown error'), 'error'); } - }; - ws.onclose = function() { renderCampaignJamPresence([], 'Disconnected; refresh Jam to reconnect.'); }; - } catch (e) { - renderCampaignJamPresence([], 'Live unavailable: ' + e.message); - } - } - - function renderCampaignJamPresence(presence, message) { - const el = document.getElementById('campaign-jam-live-status'); - if (!el) return; - if (message) { el.textContent = message; return; } - el.innerHTML = presence.length ? presence.map(function(p) { - const name = (p.actor && p.actor.name) || 'participant'; - return '<span class="campaign-jam-live-pill">' + escapeHtml(name) + (p.section ? ' · ' + escapeHtml(p.section) : '') + '</span>'; - }).join('') : 'No live participants yet.'; - } - - function focusCampaignJamLive() { - if (!_campaignJamSocket || _campaignJamSocket.readyState !== WebSocket.OPEN) return; - const section = document.getElementById('jam-live-section')?.value || ''; - _campaignJamSocket.send(JSON.stringify({ type: 'focus', section })); - } - - function submitCampaignJamLiveEdit() { - if (!_campaignJamSocket || _campaignJamSocket.readyState !== WebSocket.OPEN) { showToast('Jam live socket is not connected', 'error'); return; } - const content = document.getElementById('jam-live-content')?.value || ''; - _campaignJamSocket.send(JSON.stringify({ type: 'edit', base_revision_id: _campaignJamLiveRevision, content, reason: 'live co-edit' })); - } - - async function createCampaignJamThread(id) { - const section = document.getElementById('jam-thread-section')?.value || ''; - const body = document.getElementById('jam-thread-body')?.value || ''; - await postCampaignJam(id, 'threads', { section, body }); - } - - async function inviteCampaignJamAgent(id, thread_id) { - const prompt = await promptForJamAgent(); - if (prompt === null) return; - await postCampaignJam(id, 'agents', { thread_id, agent: 'spektacular', prompt }); - } - - function promptForJamAgent() { - return hivePrompt({title: 'Prompt for Jam agent', label: 'Prompt for the Jam agent reply and suggestion', defaultValue: 'Suggest spec text for this thread.'}); - } - - async function createCampaignJamSuggestion(id) { - const section = document.getElementById('jam-suggestion-section')?.value || ''; - const proposed_text = document.getElementById('jam-suggestion-text')?.value || ''; - await postCampaignJam(id, 'suggestions', { section, proposed_text }); - } - - async function resolveCampaignJamSuggestion(id, suggestion_id, action) { - await postCampaignJam(id, 'suggestions', { action, suggestion_id }); - } - - async function createCampaignJamPoll(id) { - const section = document.getElementById('jam-poll-section')?.value || ''; - const question = document.getElementById('jam-poll-question')?.value || ''; - const options = (document.getElementById('jam-poll-options')?.value || '').split(',').map(s => s.trim()).filter(Boolean); - await postCampaignJam(id, 'polls', { section, question, options }); - } - - async function enableCampaignJamProjectSync(id) { - const value = document.getElementById('jam-project-url')?.value || ''; - const body = value.indexOf('http') === 0 ? { action: 'enable', project_url: value } : { action: 'enable', project_id: value }; - await postCampaignJam(id, 'project-sync', body); - } - - async function syncCampaignJamProject(id) { - await postCampaignJam(id, 'project-sync', { action: 'sync' }); - } - - async function disableCampaignJamProjectSync(id) { - await postCampaignJam(id, 'project-sync', { action: 'disable' }); - } - - async function postCampaignJam(id, lane, body) { - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/jam/' + lane, { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify(body) }); - const data = await r.json(); - if (!data.ok) { showToast(data.error || 'Jam update failed', 'error'); return; } - renderCampaignJam(id, data.jam || {}); - } catch (e) { showToast('Jam error: ' + e.message, 'error'); } - } - - async function resumeCampaign(id) { - if (!id) return; - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/resume', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({ surface: 'dashboard' }) }); - const data = await r.json(); - if (!data.ok) { showToast(data.error || 'Failed to resume campaign', 'error'); return; } - if (data.state) { _inceptionState = data.state; renderInception(); } - if (data.resume_command) { await hiveAlert('Resume command:\n' + data.resume_command, 'Resume command'); } - await loadCampaigns(); - } catch (e) { showToast('Error: ' + e.message, 'error'); } - } - - async function releaseCampaign(id) { - if (!id) return; - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/release', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({}) }); - const data = await r.json(); - if (!data.ok) { showToast(data.error || 'Failed to release campaign', 'error'); return; } - await loadCampaigns(); - } catch (e) { showToast('Error: ' + e.message, 'error'); } - } - - async function reviseCampaign(id) { - if (!id) return; - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/revise', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({}) }); - const data = await r.json(); - if (!data.ok) { showToast(data.error || 'Failed to revise campaign', 'error'); return; } - await loadCampaigns(); - } catch (e) { showToast('Error: ' + e.message, 'error'); } - } - - async function recheckCampaign(id) { - if (!id) return; - try { - const r = await fetch('/api/campaigns/' + encodeURIComponent(id) + '/recheck', { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify({}) }); - const data = await r.json(); - if (!data.ok) { showToast(data.error || 'Failed to recheck campaign', 'error'); return; } - await loadCampaigns(); - } catch (e) { showToast('Error: ' + e.message, 'error'); } - } - function renderInceptionProgress(phase) { var STAGES = [ { id: 'idea', label: 'Idea', icon: '💡' }, @@ -704,13 +319,6 @@ if (_inceptionPollTimer) { dashboardClearInterval('inception-activity'); _inceptionPollTimer = null; } } - function _inceptionAuthHeaders() { - var h = {'Content-Type': 'application/json'}; - var t = localStorage.getItem('hive-token'); - if (t) h['Authorization'] = 'Bearer ' + t; - return h; - } - function _inceptionStateKey(s) { if (!s) return 'inactive'; return (s.idea_slug || '') + ':' + (s.phase || ''); diff --git a/src/pkg/dashboard/static-src/js/layout.js b/src/pkg/dashboard/static-src/js/layout.js index 4e6e3f3a67..d1e6e94ccf 100644 --- a/src/pkg/dashboard/static-src/js/layout.js +++ b/src/pkg/dashboard/static-src/js/layout.js @@ -4,7 +4,7 @@ var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','sessions-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; @@ -24,7 +24,8 @@ var dashboardKeyboardSnapshot=null; function dashboardFeatureEnabled(name){try{return !!(window._lastStatus&&window._lastStatus.features&&window._lastStatus.features[name]===true);}catch(e){return false;}} function experimentLabEnabled(){return dashboardFeatureEnabled('experiment_lab');} - function dashboardFeatureSectionHidden(id){id=dashboardAliasSectionId(id);return id==='nous-section'&&!experimentLabEnabled();} + function sessionsFeatureEnabled(){return window._sessionsEnabled===true;} + function dashboardFeatureSectionHidden(id){id=dashboardAliasSectionId(id);if(id==='sessions-section')return !sessionsFeatureEnabled();if(id==='inception-section')return sessionsFeatureEnabled();return id==='nous-section'&&!experimentLabEnabled();} function dashboardCanHideSection(id){id=dashboardAliasSectionId(id);return DASHBOARD_LAYOUT_TEMPLATE.main.indexOf(id)>=0&&DASHBOARD_HIDE_DENYLIST.indexOf(id)<0&&!dashboardFeatureSectionHidden(id);} function dashboardHiddenSectionsRead(){try{var raw=localStorage.getItem(DASHBOARD_HIDDEN_SECTIONS_KEY),arr=raw?JSON.parse(raw):[];return Array.isArray(arr)?arr.filter(function(id,i){return dashboardCanHideSection(id)&&arr.indexOf(id)===i;}):[];}catch(e){return [];}} function dashboardHiddenSectionsWrite(ids){try{var valid=(ids||[]).filter(function(id,i,a){return dashboardCanHideSection(id)&&a.indexOf(id)===i;});if(valid.length)localStorage.setItem(DASHBOARD_HIDDEN_SECTIONS_KEY,JSON.stringify(valid));else localStorage.removeItem(DASHBOARD_HIDDEN_SECTIONS_KEY);}catch(e){}dashboardUpdateLayoutMenuState();} @@ -83,7 +84,7 @@ function dashboardEnsureSidebarHandles(){dashboardSidebarNavItems().forEach(function(item){var sectionId=item.getAttribute('data-section'),label=(item.querySelector('.oc-nav-text')||item).textContent.trim()||sectionId;if(!item.querySelector(':scope > .oc-nav-drag-handle')){var handle=document.createElement('button');handle.type='button';handle.className='oc-nav-drag-handle';handle.draggable=true;handle.setAttribute('data-nav-order-grip','dashboard');handle.setAttribute('data-section',sectionId);handle.setAttribute('data-stop','1');handle.setAttribute('aria-label','Move '+label+' section');handle.setAttribute('aria-pressed','false');handle.title='Drag or focus and press Alt+↑/↓ to reorder';handle.textContent='⠿';item.insertBefore(handle,item.firstChild);}if(dashboardCanHideSection(sectionId)&&!item.querySelector(':scope > .oc-nav-hide-toggle')){var hide=document.createElement('button');hide.type='button';hide.className='oc-nav-hide-toggle';hide.setAttribute('data-action','toggleDashboardSectionHidden');hide.setAttribute('data-arg0',sectionId);hide.setAttribute('data-section',sectionId);hide.setAttribute('data-stop','1');hide.setAttribute('aria-label','Hide '+label);hide.setAttribute('aria-pressed','false');hide.title='Hide '+label;hide.textContent='⊘';item.appendChild(hide);}});dashboardApplyHiddenSections();} function dashboardSidebarLayoutFromDom(){var ids=dashboardSidebarNavItems().map(function(item){return item.getAttribute('data-section');}).filter(Boolean);return dashboardLayoutNormalize({v:DASHBOARD_LAYOUT_VERSION,main:ids});} function dashboardApplyOrderFromSidebar(){var layout=dashboardSidebarLayoutFromDom();dashboardApplyLayout(layout);dashboardLayoutWrite();dashboardLayoutAnnounce('Dashboard sidebar order saved.');} - function dashboardSortSidebar(){var order=dashboardLayoutNormalize(dashboardLayoutCurrent()).main,rank={};order.forEach(function(id,i){rank[id]=i;});var items=dashboardSidebarNavItems();items.sort(function(a,b){return rank[a.getAttribute('data-section')]-rank[b.getAttribute('data-section')];});var home=document.getElementById('dashboard-section-nav-group')||(items.length&&items[0].closest('.oc-nav-group'));if(!home)return;items.forEach(function(item){home.appendChild(item);});dashboardEnsureSidebarHandles();Array.prototype.forEach.call(document.querySelectorAll('.oc-nav-group'),function(group){if(group===home||group.id==='oc-agent-nav-group'){group.hidden=false;return;}group.hidden=group.querySelectorAll('.oc-nav-item').length===0;});} + function dashboardSortSidebar(){var order=dashboardLayoutNormalize(dashboardLayoutCurrent()).main,rank={};order.forEach(function(id,i){rank[id]=i;});var items=dashboardSidebarNavItems();var rankOf=function(el){var r=rank[el.getAttribute('data-section')];return r===undefined?order.length:r;};items.sort(function(a,b){return rankOf(a)-rankOf(b);});var home=document.getElementById('dashboard-section-nav-group')||(items.length&&items[0].closest('.oc-nav-group'));if(!home)return;items.forEach(function(item){home.appendChild(item);});dashboardEnsureSidebarHandles();Array.prototype.forEach.call(document.querySelectorAll('.oc-nav-group'),function(group){if(group===home||group.id==='oc-agent-nav-group'){group.hidden=false;return;}group.hidden=group.querySelectorAll('.oc-nav-item').length===0;});} function dashboardLayoutSlot(region,x,y,skip){var cards=Array.prototype.filter.call(region.querySelectorAll(':scope > '+DASHBOARD_LAYOUT_CARD_SELECTOR),function(c){return c!==skip;});if(!cards.length)return {region:region,before:null};var best=null,bestDist=Infinity;cards.forEach(function(card){var rect=card.getBoundingClientRect(),before=y<(rect.top+rect.height/2),anchor=before?rect.top:rect.bottom,dist=Math.abs(y-anchor)+Math.abs(x-(rect.left+rect.width/2));if(dist<bestDist){bestDist=dist;best={region:region,before:before?card:card.nextElementSibling};}});return best||{region:region,before:null};} function dashboardNearestRegion(x,y){return dashboardRegionEl('main');} function dashboardMoveIndicator(slot){var indicator=dashboardDragState&&dashboardDragState.indicator;if(!indicator||!slot||!slot.region)return;dashboardInsertSection(slot.region,indicator,slot.before);} diff --git a/src/pkg/dashboard/static-src/js/refresh.js b/src/pkg/dashboard/static-src/js/refresh.js index 4efb331c8d..76525264eb 100644 --- a/src/pkg/dashboard/static-src/js/refresh.js +++ b/src/pkg/dashboard/static-src/js/refresh.js @@ -569,7 +569,7 @@ const COLLAPSIBLE_SECTION_IDS = [ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', - 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', + 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', 'sessions-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index 671dbc836d..3c3d990ca8 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -10,7 +10,7 @@ // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', - 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', + 'contributors-section', 'inception-section', 'sessions-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section' ]); const DASHBOARD_COLLAPSE_DEFAULTS_KEY = 'hive.dashboard.collapseDefaults'; @@ -121,6 +121,7 @@ 'review-queue-subsection': { title: 'Queue', summary: '—' }, 'nous-section': { title: '🧪 Experiment Lab (nous)', summary: '—', settingsHtml: sectionSettingsGear('Experiment Lab', 'openNousConfig', 'Experiment Lab Configuration') }, 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, + 'sessions-section': { title: '💻 Sessions', summary: '0 running · 0 total', badgeHtml: '<span class="dash-card-badge">shared terminals</span>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, @@ -159,6 +160,7 @@ 'review-pipeline-subsection': { heading: 'Review Pipeline', help: 'Each open PR in the stage the review pipeline derived for it (GET /api/review/pipeline), with reviewers, P0–P3 findings, the fix-loop counter and the next action.' }, 'review-queue-subsection': { heading: 'Queue', help: 'Every open PR across the governed repos in one deterministic rank order (GET /api/review/queue), regardless of author type, with the reasons for each position.' }, 'nous-section': { heading: 'Experiment Lab', help: 'An experimental planner that tries out code changes in your repositories and records what it learned.' }, + 'sessions-section': { heading: 'Sessions', help: 'Named terminal sessions running command-line AI tools inside Hive. Everyone on this hive can watch, and people with write access can type.' }, 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, @@ -767,6 +769,8 @@ return reviewQueueCollapsedSummary(text); } else if (sectionId === 'inception-section') { inner = miniBar([{ label: 'proposals', value: n0, color: 'var(--blue)' }, { label: 'in progress', value: nums[1] || 0, color: 'var(--green)' }], text); + } else if (sectionId === 'sessions-section') { + inner = miniBar([{ label: 'running', value: n0, color: 'var(--green)' }], text); } else if (sectionId === 'contributors-section') { inner = miniAvatarDots((typeof _cachedContributors === 'undefined' ? [] : _cachedContributors), n0, text); } else if (sectionId === 'debug-section') { diff --git a/src/pkg/dashboard/static-src/js/sessions.js b/src/pkg/dashboard/static-src/js/sessions.js new file mode 100644 index 0000000000..30233bde38 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/sessions.js @@ -0,0 +1,624 @@ + // ── Sessions section ── + // Named, grouped, shared CLI-agent terminal sessions. When the spoke has + // `GET /api/sessions` -> {enabled:true} this section replaces the Inception + // wizard UI (the legacy section stays untouched when disabled). The server + // is the source of truth; localStorage only remembers the selected tab and + // collapsed groups as a convenience. Each session gets one lazily created + // ttyd iframe that is kept alive (just hidden) while another tab is shown. + const SESSIONS_API = '/api/sessions'; + const SESSIONS_POLL_MS = 5000; + const SESSIONS_LOG_POLL_MS = 3000; + const SESSIONS_LOG_LINES = 500; + const SESSIONS_ASSERTION_RENEW_MS = 600000; + const SESSIONS_TMUX_PREFIX = 'hive-sess-'; + const SESSIONS_CAMPAIGNS_TAB = 'campaigns'; + const SESSIONS_UNGROUPED = ''; + const SESSIONS_UNGROUPED_LABEL = 'Ungrouped'; + const SESSIONS_SELECTED_KEY = 'hive.sessions.selected'; + const SESSIONS_COLLAPSED_KEY = 'hive.sessions.collapsedGroups'; + const SESSIONS_WRITE_ROLES = Object.freeze(['owner', 'read-write']); + const SESSIONS_STATUS_RUNNING = 'running'; + const SESSIONS_STATUS_ENDED = 'ended'; + const SESSIONS_TYPING_NOTE = 'Number of browsers attached to this terminal. Everyone attached shares the same keyboard, and typing is not attributed to individual users.'; + const SESSIONS_HEADER_CONTENT_TYPE = 'Content-Type'; + + let _sessionsData = { enabled: false, sessions: [], vendors: [], limits: {} }; + let _sessionsRole = ''; + let _sessionsSelected = ''; + let _sessionsCollapsed = {}; + let _sessionsLoaded = false; + let _sessionsNewOpen = false; + let _sessionsLogTimerOn = false; + let _sessionsAssertionTimerOn = false; + const _sessionsFrames = {}; + const _sessionsFrameFailed = {}; + // Extension slot for the input bar: other features (for example future + // "Seed from..." providers) register {id, label, title, run(ctx)} here and + // get an entry in the bar's "More" menu. Empty registry renders no menu. + const SESSIONS_INPUT_ACTIONS = []; + + // ── Pure helpers (unit-tested) ── + function sessionsRoleCanWrite(role) { + return SESSIONS_WRITE_ROLES.indexOf(String(role || '')) >= 0; + } + function sessionsGroupLabel(group) { + return group ? String(group) : SESSIONS_UNGROUPED_LABEL; + } + function sessionsGroupSessions(sessions) { + const byGroup = {}; + (sessions || []).forEach(function(s) { + if (!s || !s.id) return; + const g = s.group || SESSIONS_UNGROUPED; + (byGroup[g] = byGroup[g] || []).push(s); + }); + const names = Object.keys(byGroup).sort(function(a, b) { + if (a === SESSIONS_UNGROUPED) return 1; + if (b === SESSIONS_UNGROUPED) return -1; + return a.localeCompare(b); + }); + return names.map(function(g) { return { group: g, label: sessionsGroupLabel(g), sessions: byGroup[g] }; }); + } + function sessionsExistingGroups(sessions) { + return sessionsGroupSessions(sessions).map(function(g) { return g.group; }).filter(Boolean); + } + function sessionsPickSelected(sessions, wanted) { + if (wanted === SESSIONS_CAMPAIGNS_TAB) return wanted; + const list = sessions || []; + if (wanted && list.some(function(s) { return s && s.id === wanted; })) return wanted; + return list.length && list[0] ? list[0].id : SESSIONS_CAMPAIGNS_TAB; + } + // Flat tab order as rendered (group order, then session order), used for arrow keys. + function sessionsTabOrder(sessions, collapsed, selected) { + const out = []; + sessionsGroupSessions(sessions).forEach(function(g) { + g.sessions.forEach(function(s) { + if (collapsed && collapsed[g.group] && s.id !== selected) return; + out.push(s.id); + }); + }); + out.push(SESSIONS_CAMPAIGNS_TAB); + return out; + } + function sessionsNextTab(order, current, key) { + if (!order || !order.length) return ''; + const i = Math.max(0, order.indexOf(current)); + if (key === 'ArrowRight' || key === 'ArrowDown') return order[(i + 1) % order.length]; + if (key === 'ArrowLeft' || key === 'ArrowUp') return order[(i + order.length - 1) % order.length]; + if (key === 'Home') return order[0]; + if (key === 'End') return order[order.length - 1]; + return ''; + } + // The terminal iframe needs terminal rights; everyone else gets the scrollback view. + function sessionsViewMode(role, session) { + if (!session) return 'none'; + if (session.status === SESSIONS_STATUS_ENDED) return 'ended'; + return sessionsRoleCanWrite(role) ? 'terminal' : 'scrollback'; + } + function sessionsByteLength(text) { + const s = String(text == null ? '' : text); + if (typeof TextEncoder === 'function') return new TextEncoder().encode(s).length; + return unescape(encodeURIComponent(s)).length; + } + function sessionsFormatBytes(n) { + const KIB = 1024; + const MIB = KIB * KIB; + if (n >= MIB) return (n / MIB).toFixed(1).replace(/\.0$/, '') + ' MiB'; + if (n >= KIB) return Math.round(n / KIB) + ' KiB'; + return n + ' bytes'; + } + function sessionsCheckInput(text, limits) { + const value = String(text == null ? '' : text); + if (!value.trim()) return { ok: false, error: 'Type or paste something to send first.' }; + const max = Number(limits && limits.max_input_bytes) || 0; + const size = sessionsByteLength(value); + if (max > 0 && size > max) { + return { ok: false, error: 'That is ' + sessionsFormatBytes(size) + '; a single send is limited to ' + sessionsFormatBytes(max) + '. Split it up or attach it as a file.' }; + } + return { ok: true, bytes: size }; + } + function sessionsCheckUpload(file, limits) { + if (!file) return { ok: false, error: 'Choose a file to attach first.' }; + const max = Number(limits && limits.max_upload_bytes) || 0; + const size = Number(file.size) || 0; + if (!size) return { ok: false, error: (file.name || 'That file') + ' is empty.' }; + if (max > 0 && size > max) { + return { ok: false, error: (file.name || 'That file') + ' is ' + sessionsFormatBytes(size) + '; uploads are limited to ' + sessionsFormatBytes(max) + '.' }; + } + return { ok: true, bytes: size }; + } + function sessionsCanCreate(role, data) { + if (!sessionsRoleCanWrite(role)) return { ok: false, reason: 'Your role can watch sessions but cannot start them.' }; + const vendors = ((data && data.vendors) || []).filter(function(v) { return v && v.available; }); + if (!vendors.length) return { ok: false, reason: 'No CLI vendor is available on this hive.' }; + const max = Number(data && data.limits && data.limits.max_sessions) || 0; + const running = ((data && data.sessions) || []).filter(function(s) { return s && s.status === SESSIONS_STATUS_RUNNING; }).length; + if (max > 0 && running >= max) return { ok: false, reason: 'This hive is at its limit of ' + max + ' running sessions. Close one first.' }; + return { ok: true, reason: '' }; + } + function sessionsTmuxName(id) { return SESSIONS_TMUX_PREFIX + id; } + function sessionsTerminalPath(id, code) { + return '/terminal/?arg=' + encodeURIComponent(sessionsTmuxName(id)) + (code ? '&code=' + encodeURIComponent(code) : ''); + } + function sessionsIsEnabled() { return window._sessionsEnabled === true; } + + // ── Storage conveniences (never required) ── + function sessionsStoreGet(key) { try { return localStorage.getItem(key) || ''; } catch (e) { return ''; } } + function sessionsStoreSet(key, value) { try { localStorage.setItem(key, value); } catch (e) { /* convenience only */ } } + function sessionsReadCollapsed() { + try { const v = JSON.parse(sessionsStoreGet(SESSIONS_COLLAPSED_KEY) || '{}'); return v && typeof v === 'object' ? v : {}; } catch (e) { return {}; } + } + + // ── Networking ── + function sessionsHeaders(json) { + const h = _inceptionAuthHeaders(); + if (!json) delete h[SESSIONS_HEADER_CONTENT_TYPE]; + return h; + } + async function sessionsApi(path, method, body) { + const opts = { method: method || 'GET', headers: sessionsHeaders(body !== undefined && !(body instanceof FormData)), credentials: 'same-origin', cache: 'no-store' }; + if (body instanceof FormData) opts.body = body; + else if (body !== undefined) opts.body = JSON.stringify(body); + const resp = await fetch(SESSIONS_API + path, opts); + let data = {}; + try { data = await resp.json(); } catch (e) { /* empty body */ } + if (!resp.ok || data.ok === false) { + const err = new Error(data.error || ('HTTP ' + resp.status)); + err.status = resp.status; + throw err; + } + return data; + } + async function sessionsLoadRole() { + try { + const resp = await fetch('/api/role'); + const data = await resp.json(); + _sessionsRole = data.role || ''; + } catch (e) { _sessionsRole = ''; } + } + async function loadSessions() { + try { + const resp = await fetch(SESSIONS_API, { headers: sessionsHeaders(false), credentials: 'same-origin', cache: 'no-store' }); + if (!resp.ok) throw new Error('HTTP ' + resp.status); + const data = await resp.json(); + _sessionsData = { + enabled: data.enabled === true, + sessions: Array.isArray(data.sessions) ? data.sessions : [], + vendors: Array.isArray(data.vendors) ? data.vendors : [], + limits: data.limits || {}, + }; + } catch (e) { + if (!_sessionsLoaded) _sessionsData = { enabled: false, sessions: [], vendors: [], limits: {} }; + else return; + } + const wasEnabled = sessionsIsEnabled(); + window._sessionsEnabled = _sessionsData.enabled; + _sessionsLoaded = true; + if (wasEnabled !== _sessionsData.enabled) sessionsApplyVisibility(); + if (!_sessionsData.enabled) { dashboardClearInterval('sessions-poll'); return; } + if (!_sessionsRole) await sessionsLoadRole(); + _sessionsSelected = sessionsPickSelected(_sessionsData.sessions, _sessionsSelected || sessionsStoreGet(SESSIONS_SELECTED_KEY)); + renderSessions(); + } + function sessionsStartPolling() { + dashboardSetInterval('sessions-poll', loadSessions, SESSIONS_POLL_MS); + } + + // ── Visibility: Sessions replaces Inception when enabled ── + function sessionsApplyVisibility() { + const on = sessionsIsEnabled(); + [['sessions-section', on], ['inception-section', !on]].forEach(function(pair) { + const el = document.getElementById(pair[0]); + if (el) el.hidden = !pair[1]; + const nav = document.querySelector('.oc-nav-item[data-section="' + pair[0] + '"]'); + if (nav) nav.hidden = !pair[1]; + }); + if (typeof dashboardApplyLayout === 'function' && typeof dashboardLayoutNormalize === 'function') { + dashboardApplyLayout(dashboardLayoutNormalize(dashboardLayoutRead())); + } + if (on) { + if (typeof refreshSectionCardShell === 'function') refreshSectionCardShell('sessions-section'); + sessionsStartPolling(); + } + } + + // ── Rendering ── + function sessionsSelectedSession() { + return (_sessionsData.sessions || []).find(function(s) { return s.id === _sessionsSelected; }) || null; + } + function sessionsTabHtml(s) { + const selected = s.id === _sessionsSelected; + const ended = s.status === SESSIONS_STATUS_ENDED; + const attached = Number(s.attached) || 0; + const badge = attached > 0 ? '<span class="sessions-attached" title="' + esc(SESSIONS_TYPING_NOTE) + '">' + attached + '</span>' : ''; + return '<button type="button" role="tab" class="sessions-tab' + (selected ? ' active' : '') + (ended ? ' ended' : '') + '" id="sessions-tab-' + esc(s.id) + '"' + + ' aria-selected="' + selected + '" aria-controls="sessions-pane-' + esc(s.id) + '" tabindex="' + (selected ? '0' : '-1') + '"' + + ' data-action="sessionsSelect" data-arg0="' + esc(s.id) + '" data-keydown-action="sessionsTabKey" data-arg-types="s,e" data-keys="*" data-arg1="">' + + '<span class="sessions-tab-dot" data-status="' + esc(s.status || '') + '"></span>' + + '<span class="sessions-tab-name">' + esc(s.name || s.id) + '</span>' + + '<span class="sessions-tab-vendor">' + esc(s.vendor || '') + '</span>' + badge + '</button>'; + } + function sessionsTabsHtml() { + const campaignsSel = _sessionsSelected === SESSIONS_CAMPAIGNS_TAB; + let html = sessionsGroupSessions(_sessionsData.sessions).map(function(g) { + const collapsed = !!_sessionsCollapsed[g.group]; + const tabs = g.sessions.filter(function(s) { return !collapsed || s.id === _sessionsSelected; }).map(sessionsTabHtml).join(''); + return '<div class="sessions-group" role="presentation">' + + '<button type="button" class="sessions-group-toggle" aria-expanded="' + !collapsed + '" data-action="sessionsToggleGroup" data-arg0="' + esc(g.group) + '" title="' + (collapsed ? 'Expand' : 'Collapse') + ' the ' + esc(g.label) + ' group">' + + '<span class="sessions-group-chevron">' + (collapsed ? '▶' : '▼') + '</span>' + esc(g.label) + ' <span class="sessions-group-count">' + g.sessions.length + '</span></button>' + + '<div class="sessions-group-tabs" role="presentation">' + tabs + '</div></div>'; + }).join(''); + html += '<div class="sessions-group" role="presentation"><div class="sessions-group-tabs" role="presentation">' + + '<button type="button" role="tab" class="sessions-tab' + (campaignsSel ? ' active' : '') + '" id="sessions-tab-' + SESSIONS_CAMPAIGNS_TAB + '" aria-selected="' + campaignsSel + '" aria-controls="sessions-pane-' + SESSIONS_CAMPAIGNS_TAB + '" tabindex="' + (campaignsSel ? '0' : '-1') + '"' + + ' data-action="sessionsSelect" data-arg0="' + SESSIONS_CAMPAIGNS_TAB + '" data-keydown-action="sessionsTabKey" data-arg-types="s,e" data-keys="*" data-arg1="">' + + '<span class="sessions-tab-name">Campaigns</span></button></div></div>'; + return html; + } + function sessionsNewFormHtml() { + const vendors = (_sessionsData.vendors || []).filter(function(v) { return v && v.available; }); + const groups = sessionsExistingGroups(_sessionsData.sessions); + return '<div class="sessions-form-row"><label for="sessions-new-name">Name</label><input id="sessions-new-name" class="campaigns-input" maxlength="80" autocomplete="off" placeholder="e.g. fix flaky test"></div>' + + '<div class="sessions-form-row"><label for="sessions-new-group">Group</label><input id="sessions-new-group" class="campaigns-input" maxlength="80" list="sessions-group-list" autocomplete="off" placeholder="optional">' + + '<datalist id="sessions-group-list">' + groups.map(function(g) { return '<option value="' + esc(g) + '"></option>'; }).join('') + '</datalist></div>' + + '<div class="sessions-form-row"><label for="sessions-new-vendor">CLI</label><select id="sessions-new-vendor" class="campaigns-input">' + + vendors.map(function(v) { return '<option value="' + esc(v.id) + '">' + esc(v.label || v.id) + '</option>'; }).join('') + '</select></div>' + + '<div class="sessions-form-actions"><button type="button" class="hv-btn btn-primary btn-sm" data-action="sessionsCreate">Start session</button>' + + '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="sessionsToggleNew">Cancel</button></div>' + + '<div class="sessions-note">Sessions run the vendor CLI with permission prompts on. Everyone with terminal rights on this hive can attach and type.</div>'; + } + function sessionsToolbarHtml() { + const can = sessionsCanCreate(_sessionsRole, _sessionsData); + return '<div class="sessions-toolbar"><div id="sessions-tabs" class="sessions-tabs" role="tablist" aria-label="Sessions">' + sessionsTabsHtml() + '</div>' + + '<button type="button" class="hv-btn btn-primary btn-sm sessions-new-btn" data-action="sessionsToggleNew"' + (can.ok ? '' : ' disabled title="' + esc(can.reason) + '"') + '>+ New session</button></div>' + + '<div id="sessions-new" class="sessions-new" role="group" aria-label="New session"' + (_sessionsNewOpen && can.ok ? '' : ' hidden') + '>' + (_sessionsNewOpen && can.ok ? sessionsNewFormHtml() : '') + '</div>'; + } + function sessionsHeaderHtml(s) { + const canManage = sessionsRoleCanWrite(_sessionsRole); + const attached = Number(s.attached) || 0; + const menu = canManage + ? '<details class="sessions-menu"><summary class="hv-btn btn-secondary btn-sm" aria-label="Session actions">⋯</summary><div class="sessions-menu-items">' + + '<button type="button" data-action="sessionsRename" data-arg0="' + esc(s.id) + '">Rename</button>' + + '<button type="button" data-action="sessionsRegroup" data-arg0="' + esc(s.id) + '">Change group</button>' + + '<button type="button" class="danger" data-action="sessionsClose" data-arg0="' + esc(s.id) + '">Close session</button></div></details>' + : ''; + return '<div class="sessions-pane-head"><span class="sessions-pane-title">' + esc(s.name || s.id) + '</span>' + + '<span class="sessions-pane-meta">' + esc(s.vendor || '') + ' · started by ' + esc(s.creator || 'unknown') + (s.group ? ' · ' + esc(s.group) : '') + '</span>' + + '<span class="sessions-attached sessions-pane-attached" title="' + esc(SESSIONS_TYPING_NOTE) + '">' + attached + ' attached</span>' + menu + '</div>'; + } + function sessionsInputBarHtml(s) { + if (!sessionsRoleCanWrite(_sessionsRole) || !s || s.status !== SESSIONS_STATUS_RUNNING) return ''; + const max = Number(_sessionsData.limits && _sessionsData.limits.max_input_bytes) || 0; + const more = SESSIONS_INPUT_ACTIONS.length + ? '<details class="sessions-menu sessions-input-more"><summary class="hv-btn btn-secondary btn-sm">More</summary><div class="sessions-menu-items">' + + SESSIONS_INPUT_ACTIONS.map(function(a) { return '<button type="button" data-action="sessionsInputAction" data-arg0="' + esc(a.id) + '" title="' + esc(a.title || '') + '">' + esc(a.label) + '</button>'; }).join('') + + '</div></details>' + : ''; + return '<div class="sessions-input" data-session="' + esc(s.id) + '">' + + '<label class="sessions-input-label" for="sessions-input-text-' + esc(s.id) + '">Send text to this session</label>' + + '<textarea id="sessions-input-text-' + esc(s.id) + '" class="sessions-input-text" rows="3" placeholder="Paste or type a prompt, then Send. Multiline is fine."' + + ' data-input-action="sessionsInputChanged" data-arg-types="t"></textarea>' + + '<div class="sessions-input-actions">' + + '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="sessionsSend" data-arg0="0" title="Type the text into the terminal without pressing Enter">Send</button>' + + '<button type="button" class="hv-btn btn-primary btn-sm" data-action="sessionsSend" data-arg0="1" title="Type the text and press Enter">Send and run</button>' + + '<label class="hv-btn btn-secondary btn-sm sessions-attach" title="Upload a file into the session workspace and type its path">Attach file' + + '<input type="file" class="sessions-file" id="sessions-file-' + esc(s.id) + '" data-change-action="sessionsFilePicked" data-arg-types="t"></label>' + more + + '<span class="sessions-input-count" id="sessions-input-count-' + esc(s.id) + '" aria-live="polite">' + (max ? '0 / ' + sessionsFormatBytes(max) : '') + '</span></div></div>'; + } + function sessionsPaneBodyHtml(s, mode) { + if (mode === 'terminal') return '<div class="sessions-terminal" data-terminal-host="' + esc(s.id) + '"></div>' + sessionsInputBarHtml(s); + let note; + if (mode === 'ended') { + const can = sessionsRoleCanWrite(_sessionsRole); + note = '<div class="sessions-note">This session has ended (the hive restarted or the process exited). Scrollback below is the last output.' + + (can ? ' <button type="button" class="hv-btn btn-primary btn-sm" data-action="sessionsResume" data-arg0="' + esc(s.id) + '">Resume</button>' : '') + '</div>'; + } else { + note = '<div class="sessions-note">Your role can watch this session but not type into it, so you see a read-only scrollback that refreshes every few seconds.</div>'; + } + return note + '<pre class="sessions-log" id="sessions-log-' + esc(s.id) + '" tabindex="0" aria-label="Session scrollback">Loading scrollback…</pre>'; + } + function sessionsEnsurePane(s) { + const host = document.getElementById('sessions-panes'); + if (!host) return null; + let pane = document.getElementById('sessions-pane-' + s.id); + const mode = sessionsViewMode(_sessionsRole, s) === 'terminal' && _sessionsFrameFailed[s.id] ? 'scrollback' : sessionsViewMode(_sessionsRole, s); + if (pane && pane.getAttribute('data-mode') !== mode) { + // Mode changed (ended, resumed, role change): rebuild this pane only. + delete _sessionsFrames[s.id]; + pane.remove(); + pane = null; + } + if (!pane) { + pane = document.createElement('div'); + pane.id = 'sessions-pane-' + s.id; + pane.className = 'sessions-pane'; + pane.setAttribute('role', 'tabpanel'); + pane.setAttribute('aria-labelledby', 'sessions-tab-' + s.id); + pane.setAttribute('data-mode', mode); + pane.innerHTML = '<div class="sessions-pane-header"></div>' + sessionsPaneBodyHtml(s, mode); + host.appendChild(pane); + } + const head = pane.querySelector('.sessions-pane-header'); + if (head) head.innerHTML = sessionsHeaderHtml(s); + return pane; + } + function renderSessions() { + const panel = document.getElementById('sessions-panel'); + if (!panel) return; + refreshSectionCardShell('sessions-section'); + _sessionsCollapsed = Object.keys(_sessionsCollapsed).length ? _sessionsCollapsed : sessionsReadCollapsed(); + const sessions = _sessionsData.sessions || []; + setSectionSummary('sessions-section', sessionsSummary(sessions)); + let root = document.getElementById('sessions-root'); + if (!root) { + panel.innerHTML = '<div id="sessions-root" class="sessions-root"><div id="sessions-chrome"></div><div id="sessions-empty" class="sessions-note" hidden></div><div id="sessions-panes" class="sessions-panes"></div></div>'; + root = document.getElementById('sessions-root'); + } + // Re-render only the chrome so live iframes and a half-typed input survive polling. + const chrome = document.getElementById('sessions-chrome'); + const focusId = document.activeElement && document.activeElement.closest && document.activeElement.closest('#sessions-tabs') ? (document.activeElement.id || '') : ''; + const typing = document.activeElement && document.activeElement.closest && document.activeElement.closest('#sessions-new'); + if (!typing) chrome.innerHTML = sessionsToolbarHtml(); + if (focusId) { const f = document.getElementById(focusId); if (f) f.focus(); } + const empty = document.getElementById('sessions-empty'); + empty.hidden = sessions.length > 0 || _sessionsSelected !== SESSIONS_CAMPAIGNS_TAB; + empty.textContent = sessions.length ? '' : 'No sessions yet. Start one with “New session” to run a CLI agent here; everyone on this hive can watch or join it.'; + const liveIds = {}; + sessions.forEach(function(s) { liveIds[s.id] = true; sessionsEnsurePane(s); }); + Array.prototype.forEach.call(document.querySelectorAll('#sessions-panes .sessions-pane'), function(pane) { + const id = pane.id.replace('sessions-pane-', ''); + if (id !== SESSIONS_CAMPAIGNS_TAB && !liveIds[id]) { delete _sessionsFrames[id]; pane.remove(); } + }); + sessionsEnsureCampaignsPane(); + sessionsShowSelected(); + } + function sessionsSummary(sessions) { + const running = (sessions || []).filter(function(s) { return s.status === SESSIONS_STATUS_RUNNING; }).length; + return running + ' running · ' + (sessions || []).length + ' total'; + } + function sessionsEnsureCampaignsPane() { + const host = document.getElementById('sessions-panes'); + if (!host || document.getElementById('sessions-pane-' + SESSIONS_CAMPAIGNS_TAB)) return; + const pane = document.createElement('div'); + pane.id = 'sessions-pane-' + SESSIONS_CAMPAIGNS_TAB; + pane.className = 'sessions-pane'; + pane.setAttribute('role', 'tabpanel'); + pane.setAttribute('aria-labelledby', 'sessions-tab-' + SESSIONS_CAMPAIGNS_TAB); + pane.setAttribute('data-mode', 'campaigns'); + host.appendChild(pane); + } + function sessionsShowSelected() { + Array.prototype.forEach.call(document.querySelectorAll('#sessions-panes .sessions-pane'), function(pane) { + pane.hidden = pane.id !== 'sessions-pane-' + _sessionsSelected; + }); + if (_sessionsSelected === SESSIONS_CAMPAIGNS_TAB) { + sessionsStopLogPoll(); + const pane = document.getElementById('sessions-pane-' + SESSIONS_CAMPAIGNS_TAB); + if (pane && !pane.firstChild) { pane.innerHTML = renderCampaignsPanel(); loadCampaigns(); } + return; + } + const s = sessionsSelectedSession(); + if (!s) return; + const mode = sessionsViewMode(_sessionsRole, s); + if (mode === 'terminal' && !_sessionsFrameFailed[s.id]) { sessionsStopLogPoll(); sessionsAttachTerminal(s); } else sessionsStartLogPoll(); + } + + // ── Terminal iframe (lazy, kept alive when hidden) ── + async function sessionsHandoff(id) { + if (typeof renewTerminalAssertion === 'function') await renewTerminalAssertion(); + const resp = await fetch('/api/terminal/handoff', { method: 'POST', headers: sessionsHeaders(true), credentials: 'same-origin', cache: 'no-store', body: JSON.stringify({ session: id }) }); + const data = await resp.json().catch(function() { return {}; }); + if (!resp.ok || !data.code) { + const err = new Error(data.error || ('HTTP ' + resp.status)); + err.status = resp.status; + throw err; + } + return data.code; + } + async function sessionsAttachTerminal(s) { + if (_sessionsFrames[s.id]) return; + const host = document.querySelector('[data-terminal-host="' + s.id + '"]'); + if (!host) return; + _sessionsFrames[s.id] = 'pending'; + try { + const code = await sessionsHandoff(s.id); + const frame = document.createElement('iframe'); + frame.className = 'sessions-frame'; + frame.title = 'Terminal for session ' + (s.name || s.id); + frame.setAttribute('allow', 'clipboard-read; clipboard-write'); + frame.src = sessionsTerminalPath(s.id, code); + host.appendChild(frame); + _sessionsFrames[s.id] = frame; + sessionsStartAssertionRenew(); + } catch (e) { + delete _sessionsFrames[s.id]; + _sessionsFrameFailed[s.id] = true; + showToast('Could not open the terminal for ' + (s.name || s.id) + ': ' + e.message + '. Showing read-only scrollback instead.', 'error'); + const pane = document.getElementById('sessions-pane-' + s.id); + if (pane) { pane.remove(); } + renderSessions(); + } + } + function sessionsHasFrames() { + return Object.keys(_sessionsFrames).some(function(k) { return _sessionsFrames[k] && _sessionsFrames[k] !== 'pending'; }); + } + function sessionsStartAssertionRenew() { + if (_sessionsAssertionTimerOn) return; + _sessionsAssertionTimerOn = true; + dashboardSetInterval('sessions-assertion', async function() { + if (!sessionsHasFrames() || !sessionsIsEnabled()) { dashboardClearInterval('sessions-assertion'); _sessionsAssertionTimerOn = false; return; } + if (typeof renewTerminalAssertion === 'function') await renewTerminalAssertion(); + }, SESSIONS_ASSERTION_RENEW_MS); + } + + // ── Scrollback view (read role / ended sessions) ── + async function sessionsRefreshLog() { + const s = sessionsSelectedSession(); + if (!s || document.hidden) return; + const el = document.getElementById('sessions-log-' + s.id); + if (!el) return; + try { + const data = await sessionsApi('/' + encodeURIComponent(s.id) + '/log?lines=' + SESSIONS_LOG_LINES, 'GET'); + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight < el.clientHeight; + el.textContent = (data.lines || []).join('\n') || '(no output yet)'; + if (atBottom) el.scrollTop = el.scrollHeight; + } catch (e) { el.textContent = 'Could not load scrollback: ' + e.message; } + } + function sessionsStartLogPoll() { + sessionsRefreshLog(); + if (_sessionsLogTimerOn) return; + _sessionsLogTimerOn = true; + dashboardSetInterval('sessions-log', sessionsRefreshLog, SESSIONS_LOG_POLL_MS); + } + function sessionsStopLogPoll() { + if (!_sessionsLogTimerOn) return; + _sessionsLogTimerOn = false; + dashboardClearInterval('sessions-log'); + } + + // ── Actions (data-action handlers) ── + function sessionsSelect(id) { + if (!id || id === _sessionsSelected) return; + _sessionsSelected = id; + sessionsStoreSet(SESSIONS_SELECTED_KEY, id); + renderSessions(); + const tab = document.getElementById('sessions-tab-' + id); + if (tab && document.activeElement && document.activeElement.closest && document.activeElement.closest('#sessions-tabs')) tab.focus(); + } + function sessionsTabKey(id, ev) { + if (!ev) return; + const order = sessionsTabOrder(_sessionsData.sessions, _sessionsCollapsed, _sessionsSelected); + const next = sessionsNextTab(order, id, ev.key); + if (!next) return; + ev.preventDefault(); + sessionsSelect(next); + const tab = document.getElementById('sessions-tab-' + next); + if (tab) tab.focus(); + } + function sessionsToggleGroup(group) { + _sessionsCollapsed[group] = !_sessionsCollapsed[group]; + sessionsStoreSet(SESSIONS_COLLAPSED_KEY, JSON.stringify(_sessionsCollapsed)); + renderSessions(); + } + function sessionsToggleNew() { + _sessionsNewOpen = !_sessionsNewOpen; + renderSessions(); + const name = document.getElementById('sessions-new-name'); + if (name) name.focus(); + } + async function sessionsCreate() { + const val = function(id) { const el = document.getElementById(id); return el ? el.value.trim() : ''; }; + const body = { name: val('sessions-new-name'), group: val('sessions-new-group'), vendor: val('sessions-new-vendor') }; + if (!body.name) { showToast('Give the session a name.', 'error'); return; } + if (!body.vendor) { showToast('Pick a CLI vendor.', 'error'); return; } + try { + const data = await sessionsApi('', 'POST', body); + _sessionsNewOpen = false; + if (data.session) { _sessionsData.sessions.push(data.session); _sessionsSelected = data.session.id; sessionsStoreSet(SESSIONS_SELECTED_KEY, data.session.id); } + renderSessions(); + showToast('Started session ' + body.name + '.', 'success'); + loadSessions(); + } catch (e) { showToast('Could not start the session: ' + e.message, 'error'); } + } + async function sessionsPatch(id, patch, okMsg) { + try { + await sessionsApi('/' + encodeURIComponent(id), 'PATCH', patch); + showToast(okMsg, 'success'); + await loadSessions(); + } catch (e) { showToast('Could not update the session: ' + e.message, 'error'); } + } + async function sessionsRename(id) { + const s = (_sessionsData.sessions || []).find(function(x) { return x.id === id; }); + if (!s) return; + const name = await hivePrompt({ title: 'Rename session', label: 'New name', defaultValue: s.name || '', confirmLabel: 'Rename', validate: function(v) { return v ? true : 'Name cannot be empty.'; } }); + if (name && name !== s.name) sessionsPatch(id, { name: name }, 'Renamed to ' + name + '.'); + } + async function sessionsRegroup(id) { + const s = (_sessionsData.sessions || []).find(function(x) { return x.id === id; }); + if (!s) return; + const options = sessionsExistingGroups(_sessionsData.sessions).map(function(g) { return { value: g, label: '' }; }); + const group = await hivePrompt({ title: 'Change group', label: 'Group (leave empty for none)', defaultValue: s.group || '', confirmLabel: 'Move', options: options }); + if (group !== null && group !== undefined && group !== false && group !== (s.group || '')) sessionsPatch(id, { group: group }, group ? 'Moved to ' + group + '.' : 'Removed from its group.'); + } + async function sessionsClose(id) { + const s = (_sessionsData.sessions || []).find(function(x) { return x.id === id; }); + if (!s) return; + const ok = await hiveConfirm({ title: 'Close this session?', message: 'Closing "' + (s.name || s.id) + '" ends the CLI process for everyone attached and discards its scrollback.', confirmLabel: 'Close session', danger: true }); + if (!ok) return; + try { + await sessionsApi('/' + encodeURIComponent(id), 'DELETE'); + delete _sessionsFrames[id]; + _sessionsData.sessions = (_sessionsData.sessions || []).filter(function(x) { return x.id !== id; }); + if (_sessionsSelected === id) _sessionsSelected = sessionsPickSelected(_sessionsData.sessions, ''); + renderSessions(); + showToast('Session closed.', 'success'); + } catch (e) { showToast('Could not close the session: ' + e.message, 'error'); } + } + async function sessionsResume(id) { + try { + await sessionsApi('/' + encodeURIComponent(id) + '/resume', 'POST', {}); + showToast('Session resumed.', 'success'); + await loadSessions(); + } catch (e) { showToast('Could not resume the session: ' + e.message, 'error'); } + } + + // ── Input bar ── + function sessionsInputChanged(el) { + const bar = el && el.closest ? el.closest('.sessions-input') : null; + if (!bar) return; + const max = Number(_sessionsData.limits && _sessionsData.limits.max_input_bytes) || 0; + const counter = bar.querySelector('.sessions-input-count'); + if (!counter || !max) return; + const size = sessionsByteLength(el.value); + counter.textContent = sessionsFormatBytes(size) + ' / ' + sessionsFormatBytes(max); + counter.classList.toggle('over', size > max); + } + async function sessionsPostInput(id, text, submit) { + await sessionsApi('/' + encodeURIComponent(id) + '/input', 'POST', { text: text, submit: !!submit }); + } + async function sessionsSend(submit) { + const s = sessionsSelectedSession(); + if (!s) return; + const box = document.getElementById('sessions-input-text-' + s.id); + if (!box) return; + const check = sessionsCheckInput(box.value, _sessionsData.limits); + if (!check.ok) { showToast(check.error, 'error'); return; } + try { + await sessionsPostInput(s.id, box.value, String(submit) === '1' || submit === true); + box.value = ''; + sessionsInputChanged(box); + showToast('Sent to ' + (s.name || s.id) + '.', 'success'); + } catch (e) { showToast('Could not send: ' + e.message, 'error'); } + } + async function sessionsFilePicked(el) { + const s = sessionsSelectedSession(); + const file = el && el.files ? el.files[0] : null; + if (!s || !file) return; + const check = sessionsCheckUpload(file, _sessionsData.limits); + if (!check.ok) { showToast(check.error, 'error'); el.value = ''; return; } + try { + const form = new FormData(); + form.append('file', file); + const data = await sessionsApi('/' + encodeURIComponent(s.id) + '/upload', 'POST', form); + if (!data.path) throw new Error('the server returned no file path'); + await sessionsPostInput(s.id, data.path, false); + showToast('Uploaded ' + file.name + ' and typed its path into the session.', 'success'); + } catch (e) { showToast('Could not attach the file: ' + e.message, 'error'); } + el.value = ''; + } + // Registry slot: features add entries to the input bar's "More" menu. + function sessionsRegisterInputAction(action) { + if (!action || !action.id || !action.label || typeof action.run !== 'function') return false; + if (SESSIONS_INPUT_ACTIONS.some(function(a) { return a.id === action.id; })) return false; + SESSIONS_INPUT_ACTIONS.push(action); + if (_sessionsLoaded) renderSessions(); + return true; + } + function sessionsInputAction(id) { + const action = SESSIONS_INPUT_ACTIONS.find(function(a) { return a.id === id; }); + const s = sessionsSelectedSession(); + if (!action || !s) return; + const box = document.getElementById('sessions-input-text-' + s.id); + action.run({ session: s, role: _sessionsRole, textarea: box, send: function(text, submit) { return sessionsPostInput(s.id, text, submit); } }); + } + document.addEventListener('DOMContentLoaded', function() { + loadSessions(); + }); diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index a9908261a7..8949c8915f 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -37,11 +37,13 @@ css/responsive.css css/recipes-utilities.css css/welcome.css css/review-campaigns.css +css/sessions.css css/kiosk.css css/popout.css css/reader.css css/variables.css glue/style-close-body-open.html +js/auth-headers.js js/reader.js js/boot.js glue/boot-script-close.html @@ -129,8 +131,10 @@ js/review-queue.js js/contributors.js js/refresh.js js/scroll-stabilizer.js +js/campaigns.js js/inception.js js/subscription-gauge.js +js/sessions.js js/debug-logs.js js/tooltips-dialogs.js js/plan-agent-controls.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index bb6b943363..f728020095 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -7738,6 +7738,56 @@ .review-priority { display: inline-block; padding: var(--sp-1) var(--sp-4); border-radius: var(--r-sm); font-size: var(--fs-2xs); color: var(--muted); border: 1px solid var(--blue); } .review-priority-high { border-color: var(--red); } .review-priority-low { border-color: var(--muted); } + /* Sessions section: tabbed, shared CLI terminals. Tokens only. */ + #sessions-section[hidden], + #inception-section[hidden], + .oc-nav-item[data-section="sessions-section"][hidden], + .oc-nav-item[data-section="inception-section"][hidden] { display: none !important; } + .sessions-root { display: flex; flex-direction: column; gap: var(--sp-4); } + .sessions-toolbar { display: flex; align-items: flex-start; gap: var(--sp-4); flex-wrap: wrap; } + .sessions-tabs { display: flex; flex-wrap: wrap; gap: var(--sp-4); flex: 1 1 auto; min-width: 0; } + .sessions-group { display: flex; flex-direction: column; gap: var(--sp-2); } + .sessions-group-toggle { background: none; border: 0; color: var(--muted); font: inherit; font-size: var(--fs-xs); text-align: left; cursor: pointer; padding: var(--sp-1) var(--sp-2); } + .sessions-group-toggle:hover { color: var(--fg); } + .sessions-group-chevron { margin-right: var(--sp-2); font-size: var(--fs-2xs); } + .sessions-group-count { opacity: 0.7; } + .sessions-group-tabs { display: flex; flex-wrap: wrap; gap: var(--sp-2); } + .sessions-tab { display: inline-flex; align-items: center; gap: var(--sp-3); background: var(--card-bg); color: var(--fg); border: 1px solid var(--border); border-radius: var(--r-sm); padding: var(--sp-3) var(--sp-4); font: inherit; font-size: var(--fs-sm); cursor: pointer; } + .sessions-tab:hover { border-color: var(--accent); } + .sessions-tab.active { border-color: var(--accent); background: color-mix(in srgb, var(--accent) 14%, transparent); font-weight: var(--fw-semibold); } + .sessions-tab.ended { opacity: 0.7; } + .sessions-tab:focus-visible, .sessions-group-toggle:focus-visible { outline: var(--focus-ring-width) solid var(--accent); outline-offset: var(--sp-1); } + .sessions-tab-dot { width: var(--status-dot-size); height: var(--status-dot-size); border-radius: var(--r-pill); background: var(--muted); } + .sessions-tab-dot[data-status="running"] { background: var(--green); } + .sessions-tab-vendor { color: var(--muted); font-size: var(--fs-xs); } + .sessions-attached { border: 1px solid var(--border); border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); color: var(--muted); font-size: var(--fs-xs); cursor: help; } + .sessions-new { border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-5); display: grid; gap: var(--sp-4); background: var(--card-bg); } + .sessions-new[hidden], .sessions-pane[hidden], #sessions-empty[hidden] { display: none; } + .sessions-form-row { display: grid; gap: var(--sp-2); } + .sessions-form-row label { font-size: var(--fs-xs); color: var(--muted); } + .sessions-form-actions, .sessions-input-actions { display: flex; flex-wrap: wrap; gap: var(--sp-3); align-items: center; } + .sessions-note { color: var(--muted); font-size: var(--fs-sm); } + .sessions-pane { display: flex; flex-direction: column; gap: var(--sp-4); } + .sessions-pane-head { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-4); } + .sessions-pane-title { font-weight: var(--fw-bold); } + .sessions-pane-meta { color: var(--muted); font-size: var(--fs-sm); flex: 1 1 auto; } + .sessions-menu { position: relative; } + .sessions-menu > summary { list-style: none; cursor: pointer; } + .sessions-menu-items { position: absolute; right: 0; z-index: var(--z-sticky); display: flex; flex-direction: column; min-width: 12em; background: var(--panel); border: 1px solid var(--border); border-radius: var(--r); box-shadow: var(--shadow-raised); padding: var(--sp-2); } + .sessions-menu-items button { background: none; border: 0; color: var(--fg); font: inherit; font-size: var(--fs-sm); text-align: left; padding: var(--sp-3) var(--sp-4); border-radius: var(--r-sm); cursor: pointer; } + .sessions-menu-items button:hover { background: color-mix(in srgb, var(--border) 30%, transparent); } + .sessions-menu-items button.danger { color: var(--red); } + .sessions-terminal { height: 65vh; min-height: 20em; border: 1px solid var(--border); border-radius: var(--r); overflow: hidden; background: var(--terminal-bg); } + .sessions-frame { width: 100%; height: 100%; border: 0; display: block; } + .sessions-log { height: 65vh; min-height: 20em; overflow: auto; margin: 0; padding: var(--sp-4); background: var(--terminal-bg); color: var(--terminal-fg, var(--fg)); border: 1px solid var(--border); border-radius: var(--r); font-family: var(--font-mono); font-size: var(--fs-xs); white-space: pre-wrap; overflow-wrap: anywhere; } + .sessions-input { display: grid; gap: var(--sp-3); } + .sessions-input-label { font-size: var(--fs-xs); color: var(--muted); } + .sessions-input-text { width: 100%; box-sizing: border-box; resize: vertical; background: var(--bg); color: var(--fg); border: 1px solid var(--border); border-radius: var(--r-sm); padding: var(--sp-4); font-family: var(--font-mono); font-size: var(--fs-sm); } + .sessions-attach { cursor: pointer; } + .sessions-file { position: absolute; width: var(--line-width); height: var(--line-width); opacity: 0; overflow: hidden; } + .sessions-attach:focus-within { outline: var(--focus-ring-width) solid var(--accent); } + .sessions-input-count { margin-left: auto; color: var(--muted); font-size: var(--fs-xs); } + .sessions-input-count.over { color: var(--red); } /* ---- kiosk: unattended display mode (#15594) ---- `?kiosk=1` (see js/boot.js) is what the headless Twitch streamer in src/deploy/stream loads. The page becomes the Hive activity monitor and @@ -7966,6 +8016,15 @@ </head> <body> <script> + // ── Shared dashboard auth headers ── + // Bearer-token header helper used across the dashboard. It lived in + // inception.js; the name is kept so existing callers do not change. + function _inceptionAuthHeaders() { + var h = {'Content-Type': 'application/json'}; + var t = localStorage.getItem('hive-token'); + if (t) h['Authorization'] = 'Bearer ' + t; + return h; + } // Code activity reader (#16475): changed regions as readable paragraphs on // the kiosk. Opt-in like Matrix and fed by the same authorized pull endpoint // (the shared SSE stream only nudges). Code is always set with textContent. @@ -8683,6 +8742,7 @@ <h2 class="config-title">🐝 Getting Started<span id="welcome-progress" class=" <a class="oc-nav-item" data-section="advisory-section" data-action="ocNavigate" data-arg0="advisory-section"><span class="oc-nav-emoji">📣</span><span class="oc-nav-text">Advisory</span></a> <a class="oc-nav-item" data-section="acmm-eval-section" data-action="ocNavigate" data-arg0="acmm-eval-section"><span class="oc-nav-emoji">🎯</span><span class="oc-nav-text">ACMM Eval</span></a> <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> + <a class="oc-nav-item" data-section="sessions-section" data-action="ocNavigate" data-arg0="sessions-section" hidden><span class="oc-nav-emoji">💻</span><span class="oc-nav-text">Sessions</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> @@ -9102,6 +9162,13 @@ <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" </div> </div> + <div class="mt-6 mb-8" id="sessions-section" data-dashboard-section="sessions-section" hidden> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="sessions-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Sessions section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 💻 Sessions</h2> + <div class="section-body"> + <div id="sessions-panel"></div> + </div> + </div> + <div class="mt-6 mb-8" id="knowledge-section" data-dashboard-section="knowledge-section"> <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="knowledge-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Knowledge section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📚 Knowledge <span id="kb-status-badge" style="font-size:0.7rem;padding:var(--sp-1) var(--sp-4);border-radius:var(--r-pill);margin-left:var(--sp-4)"></span><span id="public-knowledge-header-control" data-public-knowledge-control="header" data-stop="1"></span></h2> <div class="section-body"> @@ -9381,6 +9448,11 @@ <h3 class="faq-group-title">Runs, inception and the Experiment Lab</h3> <p>Inception is the L1 human-and-brainstorm workflow for turning an idea into facts and a scaffold. Spek is the optional runner for staged speks and plans once you are using long-running runs. Both keep authority in Hive: Spek status can advance a lease, but implement still lands through the existing PR flow. See <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/inception.md" target="_blank" rel="noopener noreferrer">Inception</a> and <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/spektacular.md" target="_blank" rel="noopener noreferrer">Spek</a>.</p> </details> + <details class="faq-q" id="faq-sessions"> + <summary>What are Sessions, and where did Inception go?</summary> + <p>When sessions are enabled on your hive, the dashboard shows a <strong>Sessions</strong> section instead of Inception: named terminal tabs where a command-line AI agent runs inside Hive, shared with everyone on the hive. People with the <code>read</code> role see a read-only scrollback. Typing in a terminal is shared and not attributed to individual people. Campaigns and Jam moved to a Campaigns tab. See <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/dashboard-sections/workflow-and-agents.md#sessions" target="_blank" rel="noopener noreferrer">Sessions</a>.</p> + </details> + <details class="faq-q" id="faq-external-engines"> <summary>What are external engines like Flue and OMP allowed to do?</summary> <p>External engines are report-only admission surfaces, not extra credentials: they receive bounded context and return receipts that Hive verifies, and they never receive target-repo write credentials or dashboard tokens. Flue and OMP are independent, default-off bindings (<code>runs.external.flue.enabled</code>, <code>runs.external.omp.enabled</code>), and dispatch additionally requires the relay to advertise <code>ext-exec/flue</code> or <code>ext-exec/omp</code>. See <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/design/external-workflow-admission.md" target="_blank" rel="noopener noreferrer">external workflow admission</a> and <a href="https://github.com/hivecommons/hive/blob/v5/src/docs/contributor-relay.md" target="_blank" rel="noopener noreferrer">contributor relay capability tokens</a>.</p> @@ -10485,7 +10557,7 @@ <h3 class="faq-group-title">Where to get help</h3> // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', - 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', + 'contributors-section', 'inception-section', 'sessions-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section' ]); const DASHBOARD_COLLAPSE_DEFAULTS_KEY = 'hive.dashboard.collapseDefaults'; @@ -10596,6 +10668,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'review-queue-subsection': { title: 'Queue', summary: '—' }, 'nous-section': { title: '🧪 Experiment Lab (nous)', summary: '—', settingsHtml: sectionSettingsGear('Experiment Lab', 'openNousConfig', 'Experiment Lab Configuration') }, 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, + 'sessions-section': { title: '💻 Sessions', summary: '0 running · 0 total', badgeHtml: '<span class="dash-card-badge">shared terminals</span>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, @@ -10634,6 +10707,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'review-pipeline-subsection': { heading: 'Review Pipeline', help: 'Each open PR in the stage the review pipeline derived for it (GET /api/review/pipeline), with reviewers, P0–P3 findings, the fix-loop counter and the next action.' }, 'review-queue-subsection': { heading: 'Queue', help: 'Every open PR across the governed repos in one deterministic rank order (GET /api/review/queue), regardless of author type, with the reasons for each position.' }, 'nous-section': { heading: 'Experiment Lab', help: 'An experimental planner that tries out code changes in your repositories and records what it learned.' }, + 'sessions-section': { heading: 'Sessions', help: 'Named terminal sessions running command-line AI tools inside Hive. Everyone on this hive can watch, and people with write access can type.' }, 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, @@ -11242,6 +11316,8 @@ <h3 class="faq-group-title">Where to get help</h3> return reviewQueueCollapsedSummary(text); } else if (sectionId === 'inception-section') { inner = miniBar([{ label: 'proposals', value: n0, color: 'var(--blue)' }, { label: 'in progress', value: nums[1] || 0, color: 'var(--green)' }], text); + } else if (sectionId === 'sessions-section') { + inner = miniBar([{ label: 'running', value: n0, color: 'var(--green)' }], text); } else if (sectionId === 'contributors-section') { inner = miniAvatarDots((typeof _cachedContributors === 'undefined' ? [] : _cachedContributors), n0, text); } else if (sectionId === 'debug-section') { @@ -12106,7 +12182,7 @@ <h3 class="faq-group-title">Where to get help</h3> var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','sessions-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; @@ -12126,7 +12202,8 @@ <h3 class="faq-group-title">Where to get help</h3> var dashboardKeyboardSnapshot=null; function dashboardFeatureEnabled(name){try{return !!(window._lastStatus&&window._lastStatus.features&&window._lastStatus.features[name]===true);}catch(e){return false;}} function experimentLabEnabled(){return dashboardFeatureEnabled('experiment_lab');} - function dashboardFeatureSectionHidden(id){id=dashboardAliasSectionId(id);return id==='nous-section'&&!experimentLabEnabled();} + function sessionsFeatureEnabled(){return window._sessionsEnabled===true;} + function dashboardFeatureSectionHidden(id){id=dashboardAliasSectionId(id);if(id==='sessions-section')return !sessionsFeatureEnabled();if(id==='inception-section')return sessionsFeatureEnabled();return id==='nous-section'&&!experimentLabEnabled();} function dashboardCanHideSection(id){id=dashboardAliasSectionId(id);return DASHBOARD_LAYOUT_TEMPLATE.main.indexOf(id)>=0&&DASHBOARD_HIDE_DENYLIST.indexOf(id)<0&&!dashboardFeatureSectionHidden(id);} function dashboardHiddenSectionsRead(){try{var raw=localStorage.getItem(DASHBOARD_HIDDEN_SECTIONS_KEY),arr=raw?JSON.parse(raw):[];return Array.isArray(arr)?arr.filter(function(id,i){return dashboardCanHideSection(id)&&arr.indexOf(id)===i;}):[];}catch(e){return [];}} function dashboardHiddenSectionsWrite(ids){try{var valid=(ids||[]).filter(function(id,i,a){return dashboardCanHideSection(id)&&a.indexOf(id)===i;});if(valid.length)localStorage.setItem(DASHBOARD_HIDDEN_SECTIONS_KEY,JSON.stringify(valid));else localStorage.removeItem(DASHBOARD_HIDDEN_SECTIONS_KEY);}catch(e){}dashboardUpdateLayoutMenuState();} @@ -12185,7 +12262,7 @@ <h3 class="faq-group-title">Where to get help</h3> function dashboardEnsureSidebarHandles(){dashboardSidebarNavItems().forEach(function(item){var sectionId=item.getAttribute('data-section'),label=(item.querySelector('.oc-nav-text')||item).textContent.trim()||sectionId;if(!item.querySelector(':scope > .oc-nav-drag-handle')){var handle=document.createElement('button');handle.type='button';handle.className='oc-nav-drag-handle';handle.draggable=true;handle.setAttribute('data-nav-order-grip','dashboard');handle.setAttribute('data-section',sectionId);handle.setAttribute('data-stop','1');handle.setAttribute('aria-label','Move '+label+' section');handle.setAttribute('aria-pressed','false');handle.title='Drag or focus and press Alt+↑/↓ to reorder';handle.textContent='⠿';item.insertBefore(handle,item.firstChild);}if(dashboardCanHideSection(sectionId)&&!item.querySelector(':scope > .oc-nav-hide-toggle')){var hide=document.createElement('button');hide.type='button';hide.className='oc-nav-hide-toggle';hide.setAttribute('data-action','toggleDashboardSectionHidden');hide.setAttribute('data-arg0',sectionId);hide.setAttribute('data-section',sectionId);hide.setAttribute('data-stop','1');hide.setAttribute('aria-label','Hide '+label);hide.setAttribute('aria-pressed','false');hide.title='Hide '+label;hide.textContent='⊘';item.appendChild(hide);}});dashboardApplyHiddenSections();} function dashboardSidebarLayoutFromDom(){var ids=dashboardSidebarNavItems().map(function(item){return item.getAttribute('data-section');}).filter(Boolean);return dashboardLayoutNormalize({v:DASHBOARD_LAYOUT_VERSION,main:ids});} function dashboardApplyOrderFromSidebar(){var layout=dashboardSidebarLayoutFromDom();dashboardApplyLayout(layout);dashboardLayoutWrite();dashboardLayoutAnnounce('Dashboard sidebar order saved.');} - function dashboardSortSidebar(){var order=dashboardLayoutNormalize(dashboardLayoutCurrent()).main,rank={};order.forEach(function(id,i){rank[id]=i;});var items=dashboardSidebarNavItems();items.sort(function(a,b){return rank[a.getAttribute('data-section')]-rank[b.getAttribute('data-section')];});var home=document.getElementById('dashboard-section-nav-group')||(items.length&&items[0].closest('.oc-nav-group'));if(!home)return;items.forEach(function(item){home.appendChild(item);});dashboardEnsureSidebarHandles();Array.prototype.forEach.call(document.querySelectorAll('.oc-nav-group'),function(group){if(group===home||group.id==='oc-agent-nav-group'){group.hidden=false;return;}group.hidden=group.querySelectorAll('.oc-nav-item').length===0;});} + function dashboardSortSidebar(){var order=dashboardLayoutNormalize(dashboardLayoutCurrent()).main,rank={};order.forEach(function(id,i){rank[id]=i;});var items=dashboardSidebarNavItems();var rankOf=function(el){var r=rank[el.getAttribute('data-section')];return r===undefined?order.length:r;};items.sort(function(a,b){return rankOf(a)-rankOf(b);});var home=document.getElementById('dashboard-section-nav-group')||(items.length&&items[0].closest('.oc-nav-group'));if(!home)return;items.forEach(function(item){home.appendChild(item);});dashboardEnsureSidebarHandles();Array.prototype.forEach.call(document.querySelectorAll('.oc-nav-group'),function(group){if(group===home||group.id==='oc-agent-nav-group'){group.hidden=false;return;}group.hidden=group.querySelectorAll('.oc-nav-item').length===0;});} function dashboardLayoutSlot(region,x,y,skip){var cards=Array.prototype.filter.call(region.querySelectorAll(':scope > '+DASHBOARD_LAYOUT_CARD_SELECTOR),function(c){return c!==skip;});if(!cards.length)return {region:region,before:null};var best=null,bestDist=Infinity;cards.forEach(function(card){var rect=card.getBoundingClientRect(),before=y<(rect.top+rect.height/2),anchor=before?rect.top:rect.bottom,dist=Math.abs(y-anchor)+Math.abs(x-(rect.left+rect.width/2));if(dist<bestDist){bestDist=dist;best={region:region,before:before?card:card.nextElementSibling};}});return best||{region:region,before:null};} function dashboardNearestRegion(x,y){return dashboardRegionEl('main');} function dashboardMoveIndicator(slot){var indicator=dashboardDragState&&dashboardDragState.indicator;if(!indicator||!slot||!slot.region)return;dashboardInsertSection(slot.region,indicator,slot.before);} @@ -12670,6 +12747,8 @@ <h3 class="faq-group-title">Where to get help</h3> } function ocNavigate(section, skipScroll) { section = dashboardAliasSectionId(section); + // Sessions replaces Inception when enabled: old links and the welcome checklist land there. + if (section === 'inception-section' && window._sessionsEnabled === true) section = 'sessions-section'; if (dashboardFeatureSectionHidden(section)) return; if (dashboardSectionHidden(section)) dashboardSetSectionHidden(section,false); _ocSelectedAgent = null; @@ -49122,7 +49201,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const COLLAPSIBLE_SECTION_IDS = [ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', - 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', + 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', 'sessions-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; @@ -49270,21 +49349,13 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> } else { initScrollStabilizer(); } - // ── Inception section ── - let _inceptionState = null; + // ── Campaigns + Jam (Spek / inception campaign archive) ── + // Moved verbatim out of inception.js so the Campaigns and Jam UI survives + // the Inception wizard being replaced by the Sessions section. Plain + // top-level declarations, shared with inception.js through the single + // concatenated dashboard script; keep this file adjacent to it in manifest.txt. let _campaigns = []; - - async function loadInceptionState() { - try { - const r = await fetch('/api/inception/state', {headers: _inceptionAuthHeaders()}); - const data = await r.json(); - _inceptionState = data.active ? data.state : null; - renderInception(); - loadCampaigns(); - } catch (e) { console.error('inception state load failed', e); } - } - async function loadCampaigns() { try { const params = new URLSearchParams(); @@ -49302,58 +49373,6 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> } catch (e) { console.error('campaign load failed', e); } } - function renderInception() { - refreshSectionCardShell('inception-section'); - setSectionSummary('inception-section', inceptionSectionSummary()); - const panel = document.getElementById('inception-panel'); - if (!panel) return; - - let wizard = ''; - if (!_inceptionState || !_inceptionState.phase) { - wizard = renderInceptionModeSelect(); - } else { - switch (_inceptionState.phase) { - case 'capture': - if (_inceptionState.idea_text) { - wizard = renderInceptionProgress('idea') + renderInceptionWorking(); - } else { - wizard = _inceptionState.mode === 'greenfield' - ? renderInceptionCapture() - : renderInceptionScanProgress(); - } - break; - case 'clarify': - wizard = renderInceptionProgress('clarify') + renderInceptionClarify(); - break; - case 'structure': - wizard = renderInceptionProgress('structure') + renderInceptionStructure(); - break; - case 'scaffold': - wizard = renderInceptionProgress('scaffold') + renderInceptionScaffold(); - break; - case 'complete': - wizard = renderInceptionProgress('complete') + renderInceptionComplete(); - setTimeout(loadInceptionCompleteFiles, 100); - break; - default: - wizard = renderInceptionModeSelect(); - } - } - // Rebuilding the panel resets its own scroll and the campaign filters; - // carry both across so refreshes don't move the reader (#11032). - const panelScrollTop = panel.scrollTop; - const campaignFilters = {}; - panel.querySelectorAll('.campaigns-filters input').forEach(function(el) { campaignFilters[el.id] = el.value; }); - panel.innerHTML = '<div class="inception-spektacular-note">Project inception runs on <a href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer">Spektacular</a>, a Hive Commons project.</div>' + renderCampaignsPanel() + '<div class="campaigns-card">' + wizard + '</div>'; - Object.keys(campaignFilters).forEach(function(id) { - const el = document.getElementById(id); - if (el) { el.defaultValue = campaignFilters[id]; el.value = campaignFilters[id]; } - }); - panel.scrollTop = panelScrollTop; - renderCampaignsList(); - if (!_inceptionPollTimer) startInceptionPoll(); - } - function renderCampaignsPanel() { return ` <div class="campaigns-card"> @@ -49720,6 +49739,71 @@ <h4>GitHub Projects sync</h4> await loadCampaigns(); } catch (e) { showToast('Error: ' + e.message, 'error'); } } + // ── Inception section ── + let _inceptionState = null; + + + async function loadInceptionState() { + try { + const r = await fetch('/api/inception/state', {headers: _inceptionAuthHeaders()}); + const data = await r.json(); + _inceptionState = data.active ? data.state : null; + renderInception(); + loadCampaigns(); + } catch (e) { console.error('inception state load failed', e); } + } + + function renderInception() { + refreshSectionCardShell('inception-section'); + setSectionSummary('inception-section', inceptionSectionSummary()); + const panel = document.getElementById('inception-panel'); + if (!panel) return; + + let wizard = ''; + if (!_inceptionState || !_inceptionState.phase) { + wizard = renderInceptionModeSelect(); + } else { + switch (_inceptionState.phase) { + case 'capture': + if (_inceptionState.idea_text) { + wizard = renderInceptionProgress('idea') + renderInceptionWorking(); + } else { + wizard = _inceptionState.mode === 'greenfield' + ? renderInceptionCapture() + : renderInceptionScanProgress(); + } + break; + case 'clarify': + wizard = renderInceptionProgress('clarify') + renderInceptionClarify(); + break; + case 'structure': + wizard = renderInceptionProgress('structure') + renderInceptionStructure(); + break; + case 'scaffold': + wizard = renderInceptionProgress('scaffold') + renderInceptionScaffold(); + break; + case 'complete': + wizard = renderInceptionProgress('complete') + renderInceptionComplete(); + setTimeout(loadInceptionCompleteFiles, 100); + break; + default: + wizard = renderInceptionModeSelect(); + } + } + // Rebuilding the panel resets its own scroll and the campaign filters; + // carry both across so refreshes don't move the reader (#11032). + const panelScrollTop = panel.scrollTop; + const campaignFilters = {}; + panel.querySelectorAll('.campaigns-filters input').forEach(function(el) { campaignFilters[el.id] = el.value; }); + panel.innerHTML = '<div class="inception-spektacular-note">Project inception runs on <a href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer">Spektacular</a>, a Hive Commons project.</div>' + renderCampaignsPanel() + '<div class="campaigns-card">' + wizard + '</div>'; + Object.keys(campaignFilters).forEach(function(id) { + const el = document.getElementById(id); + if (el) { el.defaultValue = campaignFilters[id]; el.value = campaignFilters[id]; } + }); + panel.scrollTop = panelScrollTop; + renderCampaignsList(); + if (!_inceptionPollTimer) startInceptionPoll(); + } function renderInceptionProgress(phase) { var STAGES = [ @@ -49976,13 +50060,6 @@ <h4>GitHub Projects sync</h4> if (_inceptionPollTimer) { dashboardClearInterval('inception-activity'); _inceptionPollTimer = null; } } - function _inceptionAuthHeaders() { - var h = {'Content-Type': 'application/json'}; - var t = localStorage.getItem('hive-token'); - if (t) h['Authorization'] = 'Bearer ' + t; - return h; - } - function _inceptionStateKey(s) { if (!s) return 'inactive'; return (s.idea_slug || '') + ':' + (s.phase || ''); @@ -50811,6 +50888,630 @@ <h4>GitHub Projects sync</h4> showToast('The usage gauge is hidden on the Overview. Turn on "Show Claude usage gauge" in Overview settings to see every limit.', 'info'); } } + // ── Sessions section ── + // Named, grouped, shared CLI-agent terminal sessions. When the spoke has + // `GET /api/sessions` -> {enabled:true} this section replaces the Inception + // wizard UI (the legacy section stays untouched when disabled). The server + // is the source of truth; localStorage only remembers the selected tab and + // collapsed groups as a convenience. Each session gets one lazily created + // ttyd iframe that is kept alive (just hidden) while another tab is shown. + const SESSIONS_API = '/api/sessions'; + const SESSIONS_POLL_MS = 5000; + const SESSIONS_LOG_POLL_MS = 3000; + const SESSIONS_LOG_LINES = 500; + const SESSIONS_ASSERTION_RENEW_MS = 600000; + const SESSIONS_TMUX_PREFIX = 'hive-sess-'; + const SESSIONS_CAMPAIGNS_TAB = 'campaigns'; + const SESSIONS_UNGROUPED = ''; + const SESSIONS_UNGROUPED_LABEL = 'Ungrouped'; + const SESSIONS_SELECTED_KEY = 'hive.sessions.selected'; + const SESSIONS_COLLAPSED_KEY = 'hive.sessions.collapsedGroups'; + const SESSIONS_WRITE_ROLES = Object.freeze(['owner', 'read-write']); + const SESSIONS_STATUS_RUNNING = 'running'; + const SESSIONS_STATUS_ENDED = 'ended'; + const SESSIONS_TYPING_NOTE = 'Number of browsers attached to this terminal. Everyone attached shares the same keyboard, and typing is not attributed to individual users.'; + const SESSIONS_HEADER_CONTENT_TYPE = 'Content-Type'; + + let _sessionsData = { enabled: false, sessions: [], vendors: [], limits: {} }; + let _sessionsRole = ''; + let _sessionsSelected = ''; + let _sessionsCollapsed = {}; + let _sessionsLoaded = false; + let _sessionsNewOpen = false; + let _sessionsLogTimerOn = false; + let _sessionsAssertionTimerOn = false; + const _sessionsFrames = {}; + const _sessionsFrameFailed = {}; + // Extension slot for the input bar: other features (for example future + // "Seed from..." providers) register {id, label, title, run(ctx)} here and + // get an entry in the bar's "More" menu. Empty registry renders no menu. + const SESSIONS_INPUT_ACTIONS = []; + + // ── Pure helpers (unit-tested) ── + function sessionsRoleCanWrite(role) { + return SESSIONS_WRITE_ROLES.indexOf(String(role || '')) >= 0; + } + function sessionsGroupLabel(group) { + return group ? String(group) : SESSIONS_UNGROUPED_LABEL; + } + function sessionsGroupSessions(sessions) { + const byGroup = {}; + (sessions || []).forEach(function(s) { + if (!s || !s.id) return; + const g = s.group || SESSIONS_UNGROUPED; + (byGroup[g] = byGroup[g] || []).push(s); + }); + const names = Object.keys(byGroup).sort(function(a, b) { + if (a === SESSIONS_UNGROUPED) return 1; + if (b === SESSIONS_UNGROUPED) return -1; + return a.localeCompare(b); + }); + return names.map(function(g) { return { group: g, label: sessionsGroupLabel(g), sessions: byGroup[g] }; }); + } + function sessionsExistingGroups(sessions) { + return sessionsGroupSessions(sessions).map(function(g) { return g.group; }).filter(Boolean); + } + function sessionsPickSelected(sessions, wanted) { + if (wanted === SESSIONS_CAMPAIGNS_TAB) return wanted; + const list = sessions || []; + if (wanted && list.some(function(s) { return s && s.id === wanted; })) return wanted; + return list.length && list[0] ? list[0].id : SESSIONS_CAMPAIGNS_TAB; + } + // Flat tab order as rendered (group order, then session order), used for arrow keys. + function sessionsTabOrder(sessions, collapsed, selected) { + const out = []; + sessionsGroupSessions(sessions).forEach(function(g) { + g.sessions.forEach(function(s) { + if (collapsed && collapsed[g.group] && s.id !== selected) return; + out.push(s.id); + }); + }); + out.push(SESSIONS_CAMPAIGNS_TAB); + return out; + } + function sessionsNextTab(order, current, key) { + if (!order || !order.length) return ''; + const i = Math.max(0, order.indexOf(current)); + if (key === 'ArrowRight' || key === 'ArrowDown') return order[(i + 1) % order.length]; + if (key === 'ArrowLeft' || key === 'ArrowUp') return order[(i + order.length - 1) % order.length]; + if (key === 'Home') return order[0]; + if (key === 'End') return order[order.length - 1]; + return ''; + } + // The terminal iframe needs terminal rights; everyone else gets the scrollback view. + function sessionsViewMode(role, session) { + if (!session) return 'none'; + if (session.status === SESSIONS_STATUS_ENDED) return 'ended'; + return sessionsRoleCanWrite(role) ? 'terminal' : 'scrollback'; + } + function sessionsByteLength(text) { + const s = String(text == null ? '' : text); + if (typeof TextEncoder === 'function') return new TextEncoder().encode(s).length; + return unescape(encodeURIComponent(s)).length; + } + function sessionsFormatBytes(n) { + const KIB = 1024; + const MIB = KIB * KIB; + if (n >= MIB) return (n / MIB).toFixed(1).replace(/\.0$/, '') + ' MiB'; + if (n >= KIB) return Math.round(n / KIB) + ' KiB'; + return n + ' bytes'; + } + function sessionsCheckInput(text, limits) { + const value = String(text == null ? '' : text); + if (!value.trim()) return { ok: false, error: 'Type or paste something to send first.' }; + const max = Number(limits && limits.max_input_bytes) || 0; + const size = sessionsByteLength(value); + if (max > 0 && size > max) { + return { ok: false, error: 'That is ' + sessionsFormatBytes(size) + '; a single send is limited to ' + sessionsFormatBytes(max) + '. Split it up or attach it as a file.' }; + } + return { ok: true, bytes: size }; + } + function sessionsCheckUpload(file, limits) { + if (!file) return { ok: false, error: 'Choose a file to attach first.' }; + const max = Number(limits && limits.max_upload_bytes) || 0; + const size = Number(file.size) || 0; + if (!size) return { ok: false, error: (file.name || 'That file') + ' is empty.' }; + if (max > 0 && size > max) { + return { ok: false, error: (file.name || 'That file') + ' is ' + sessionsFormatBytes(size) + '; uploads are limited to ' + sessionsFormatBytes(max) + '.' }; + } + return { ok: true, bytes: size }; + } + function sessionsCanCreate(role, data) { + if (!sessionsRoleCanWrite(role)) return { ok: false, reason: 'Your role can watch sessions but cannot start them.' }; + const vendors = ((data && data.vendors) || []).filter(function(v) { return v && v.available; }); + if (!vendors.length) return { ok: false, reason: 'No CLI vendor is available on this hive.' }; + const max = Number(data && data.limits && data.limits.max_sessions) || 0; + const running = ((data && data.sessions) || []).filter(function(s) { return s && s.status === SESSIONS_STATUS_RUNNING; }).length; + if (max > 0 && running >= max) return { ok: false, reason: 'This hive is at its limit of ' + max + ' running sessions. Close one first.' }; + return { ok: true, reason: '' }; + } + function sessionsTmuxName(id) { return SESSIONS_TMUX_PREFIX + id; } + function sessionsTerminalPath(id, code) { + return '/terminal/?arg=' + encodeURIComponent(sessionsTmuxName(id)) + (code ? '&code=' + encodeURIComponent(code) : ''); + } + function sessionsIsEnabled() { return window._sessionsEnabled === true; } + + // ── Storage conveniences (never required) ── + function sessionsStoreGet(key) { try { return localStorage.getItem(key) || ''; } catch (e) { return ''; } } + function sessionsStoreSet(key, value) { try { localStorage.setItem(key, value); } catch (e) { /* convenience only */ } } + function sessionsReadCollapsed() { + try { const v = JSON.parse(sessionsStoreGet(SESSIONS_COLLAPSED_KEY) || '{}'); return v && typeof v === 'object' ? v : {}; } catch (e) { return {}; } + } + + // ── Networking ── + function sessionsHeaders(json) { + const h = _inceptionAuthHeaders(); + if (!json) delete h[SESSIONS_HEADER_CONTENT_TYPE]; + return h; + } + async function sessionsApi(path, method, body) { + const opts = { method: method || 'GET', headers: sessionsHeaders(body !== undefined && !(body instanceof FormData)), credentials: 'same-origin', cache: 'no-store' }; + if (body instanceof FormData) opts.body = body; + else if (body !== undefined) opts.body = JSON.stringify(body); + const resp = await fetch(SESSIONS_API + path, opts); + let data = {}; + try { data = await resp.json(); } catch (e) { /* empty body */ } + if (!resp.ok || data.ok === false) { + const err = new Error(data.error || ('HTTP ' + resp.status)); + err.status = resp.status; + throw err; + } + return data; + } + async function sessionsLoadRole() { + try { + const resp = await fetch('/api/role'); + const data = await resp.json(); + _sessionsRole = data.role || ''; + } catch (e) { _sessionsRole = ''; } + } + async function loadSessions() { + try { + const resp = await fetch(SESSIONS_API, { headers: sessionsHeaders(false), credentials: 'same-origin', cache: 'no-store' }); + if (!resp.ok) throw new Error('HTTP ' + resp.status); + const data = await resp.json(); + _sessionsData = { + enabled: data.enabled === true, + sessions: Array.isArray(data.sessions) ? data.sessions : [], + vendors: Array.isArray(data.vendors) ? data.vendors : [], + limits: data.limits || {}, + }; + } catch (e) { + if (!_sessionsLoaded) _sessionsData = { enabled: false, sessions: [], vendors: [], limits: {} }; + else return; + } + const wasEnabled = sessionsIsEnabled(); + window._sessionsEnabled = _sessionsData.enabled; + _sessionsLoaded = true; + if (wasEnabled !== _sessionsData.enabled) sessionsApplyVisibility(); + if (!_sessionsData.enabled) { dashboardClearInterval('sessions-poll'); return; } + if (!_sessionsRole) await sessionsLoadRole(); + _sessionsSelected = sessionsPickSelected(_sessionsData.sessions, _sessionsSelected || sessionsStoreGet(SESSIONS_SELECTED_KEY)); + renderSessions(); + } + function sessionsStartPolling() { + dashboardSetInterval('sessions-poll', loadSessions, SESSIONS_POLL_MS); + } + + // ── Visibility: Sessions replaces Inception when enabled ── + function sessionsApplyVisibility() { + const on = sessionsIsEnabled(); + [['sessions-section', on], ['inception-section', !on]].forEach(function(pair) { + const el = document.getElementById(pair[0]); + if (el) el.hidden = !pair[1]; + const nav = document.querySelector('.oc-nav-item[data-section="' + pair[0] + '"]'); + if (nav) nav.hidden = !pair[1]; + }); + if (typeof dashboardApplyLayout === 'function' && typeof dashboardLayoutNormalize === 'function') { + dashboardApplyLayout(dashboardLayoutNormalize(dashboardLayoutRead())); + } + if (on) { + if (typeof refreshSectionCardShell === 'function') refreshSectionCardShell('sessions-section'); + sessionsStartPolling(); + } + } + + // ── Rendering ── + function sessionsSelectedSession() { + return (_sessionsData.sessions || []).find(function(s) { return s.id === _sessionsSelected; }) || null; + } + function sessionsTabHtml(s) { + const selected = s.id === _sessionsSelected; + const ended = s.status === SESSIONS_STATUS_ENDED; + const attached = Number(s.attached) || 0; + const badge = attached > 0 ? '<span class="sessions-attached" title="' + esc(SESSIONS_TYPING_NOTE) + '">' + attached + '</span>' : ''; + return '<button type="button" role="tab" class="sessions-tab' + (selected ? ' active' : '') + (ended ? ' ended' : '') + '" id="sessions-tab-' + esc(s.id) + '"' + + ' aria-selected="' + selected + '" aria-controls="sessions-pane-' + esc(s.id) + '" tabindex="' + (selected ? '0' : '-1') + '"' + + ' data-action="sessionsSelect" data-arg0="' + esc(s.id) + '" data-keydown-action="sessionsTabKey" data-arg-types="s,e" data-keys="*" data-arg1="">' + + '<span class="sessions-tab-dot" data-status="' + esc(s.status || '') + '"></span>' + + '<span class="sessions-tab-name">' + esc(s.name || s.id) + '</span>' + + '<span class="sessions-tab-vendor">' + esc(s.vendor || '') + '</span>' + badge + '</button>'; + } + function sessionsTabsHtml() { + const campaignsSel = _sessionsSelected === SESSIONS_CAMPAIGNS_TAB; + let html = sessionsGroupSessions(_sessionsData.sessions).map(function(g) { + const collapsed = !!_sessionsCollapsed[g.group]; + const tabs = g.sessions.filter(function(s) { return !collapsed || s.id === _sessionsSelected; }).map(sessionsTabHtml).join(''); + return '<div class="sessions-group" role="presentation">' + + '<button type="button" class="sessions-group-toggle" aria-expanded="' + !collapsed + '" data-action="sessionsToggleGroup" data-arg0="' + esc(g.group) + '" title="' + (collapsed ? 'Expand' : 'Collapse') + ' the ' + esc(g.label) + ' group">' + + '<span class="sessions-group-chevron">' + (collapsed ? '▶' : '▼') + '</span>' + esc(g.label) + ' <span class="sessions-group-count">' + g.sessions.length + '</span></button>' + + '<div class="sessions-group-tabs" role="presentation">' + tabs + '</div></div>'; + }).join(''); + html += '<div class="sessions-group" role="presentation"><div class="sessions-group-tabs" role="presentation">' + + '<button type="button" role="tab" class="sessions-tab' + (campaignsSel ? ' active' : '') + '" id="sessions-tab-' + SESSIONS_CAMPAIGNS_TAB + '" aria-selected="' + campaignsSel + '" aria-controls="sessions-pane-' + SESSIONS_CAMPAIGNS_TAB + '" tabindex="' + (campaignsSel ? '0' : '-1') + '"' + + ' data-action="sessionsSelect" data-arg0="' + SESSIONS_CAMPAIGNS_TAB + '" data-keydown-action="sessionsTabKey" data-arg-types="s,e" data-keys="*" data-arg1="">' + + '<span class="sessions-tab-name">Campaigns</span></button></div></div>'; + return html; + } + function sessionsNewFormHtml() { + const vendors = (_sessionsData.vendors || []).filter(function(v) { return v && v.available; }); + const groups = sessionsExistingGroups(_sessionsData.sessions); + return '<div class="sessions-form-row"><label for="sessions-new-name">Name</label><input id="sessions-new-name" class="campaigns-input" maxlength="80" autocomplete="off" placeholder="e.g. fix flaky test"></div>' + + '<div class="sessions-form-row"><label for="sessions-new-group">Group</label><input id="sessions-new-group" class="campaigns-input" maxlength="80" list="sessions-group-list" autocomplete="off" placeholder="optional">' + + '<datalist id="sessions-group-list">' + groups.map(function(g) { return '<option value="' + esc(g) + '"></option>'; }).join('') + '</datalist></div>' + + '<div class="sessions-form-row"><label for="sessions-new-vendor">CLI</label><select id="sessions-new-vendor" class="campaigns-input">' + + vendors.map(function(v) { return '<option value="' + esc(v.id) + '">' + esc(v.label || v.id) + '</option>'; }).join('') + '</select></div>' + + '<div class="sessions-form-actions"><button type="button" class="hv-btn btn-primary btn-sm" data-action="sessionsCreate">Start session</button>' + + '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="sessionsToggleNew">Cancel</button></div>' + + '<div class="sessions-note">Sessions run the vendor CLI with permission prompts on. Everyone with terminal rights on this hive can attach and type.</div>'; + } + function sessionsToolbarHtml() { + const can = sessionsCanCreate(_sessionsRole, _sessionsData); + return '<div class="sessions-toolbar"><div id="sessions-tabs" class="sessions-tabs" role="tablist" aria-label="Sessions">' + sessionsTabsHtml() + '</div>' + + '<button type="button" class="hv-btn btn-primary btn-sm sessions-new-btn" data-action="sessionsToggleNew"' + (can.ok ? '' : ' disabled title="' + esc(can.reason) + '"') + '>+ New session</button></div>' + + '<div id="sessions-new" class="sessions-new" role="group" aria-label="New session"' + (_sessionsNewOpen && can.ok ? '' : ' hidden') + '>' + (_sessionsNewOpen && can.ok ? sessionsNewFormHtml() : '') + '</div>'; + } + function sessionsHeaderHtml(s) { + const canManage = sessionsRoleCanWrite(_sessionsRole); + const attached = Number(s.attached) || 0; + const menu = canManage + ? '<details class="sessions-menu"><summary class="hv-btn btn-secondary btn-sm" aria-label="Session actions">⋯</summary><div class="sessions-menu-items">' + + '<button type="button" data-action="sessionsRename" data-arg0="' + esc(s.id) + '">Rename</button>' + + '<button type="button" data-action="sessionsRegroup" data-arg0="' + esc(s.id) + '">Change group</button>' + + '<button type="button" class="danger" data-action="sessionsClose" data-arg0="' + esc(s.id) + '">Close session</button></div></details>' + : ''; + return '<div class="sessions-pane-head"><span class="sessions-pane-title">' + esc(s.name || s.id) + '</span>' + + '<span class="sessions-pane-meta">' + esc(s.vendor || '') + ' · started by ' + esc(s.creator || 'unknown') + (s.group ? ' · ' + esc(s.group) : '') + '</span>' + + '<span class="sessions-attached sessions-pane-attached" title="' + esc(SESSIONS_TYPING_NOTE) + '">' + attached + ' attached</span>' + menu + '</div>'; + } + function sessionsInputBarHtml(s) { + if (!sessionsRoleCanWrite(_sessionsRole) || !s || s.status !== SESSIONS_STATUS_RUNNING) return ''; + const max = Number(_sessionsData.limits && _sessionsData.limits.max_input_bytes) || 0; + const more = SESSIONS_INPUT_ACTIONS.length + ? '<details class="sessions-menu sessions-input-more"><summary class="hv-btn btn-secondary btn-sm">More</summary><div class="sessions-menu-items">' + + SESSIONS_INPUT_ACTIONS.map(function(a) { return '<button type="button" data-action="sessionsInputAction" data-arg0="' + esc(a.id) + '" title="' + esc(a.title || '') + '">' + esc(a.label) + '</button>'; }).join('') + + '</div></details>' + : ''; + return '<div class="sessions-input" data-session="' + esc(s.id) + '">' + + '<label class="sessions-input-label" for="sessions-input-text-' + esc(s.id) + '">Send text to this session</label>' + + '<textarea id="sessions-input-text-' + esc(s.id) + '" class="sessions-input-text" rows="3" placeholder="Paste or type a prompt, then Send. Multiline is fine."' + + ' data-input-action="sessionsInputChanged" data-arg-types="t"></textarea>' + + '<div class="sessions-input-actions">' + + '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="sessionsSend" data-arg0="0" title="Type the text into the terminal without pressing Enter">Send</button>' + + '<button type="button" class="hv-btn btn-primary btn-sm" data-action="sessionsSend" data-arg0="1" title="Type the text and press Enter">Send and run</button>' + + '<label class="hv-btn btn-secondary btn-sm sessions-attach" title="Upload a file into the session workspace and type its path">Attach file' + + '<input type="file" class="sessions-file" id="sessions-file-' + esc(s.id) + '" data-change-action="sessionsFilePicked" data-arg-types="t"></label>' + more + + '<span class="sessions-input-count" id="sessions-input-count-' + esc(s.id) + '" aria-live="polite">' + (max ? '0 / ' + sessionsFormatBytes(max) : '') + '</span></div></div>'; + } + function sessionsPaneBodyHtml(s, mode) { + if (mode === 'terminal') return '<div class="sessions-terminal" data-terminal-host="' + esc(s.id) + '"></div>' + sessionsInputBarHtml(s); + let note; + if (mode === 'ended') { + const can = sessionsRoleCanWrite(_sessionsRole); + note = '<div class="sessions-note">This session has ended (the hive restarted or the process exited). Scrollback below is the last output.' + + (can ? ' <button type="button" class="hv-btn btn-primary btn-sm" data-action="sessionsResume" data-arg0="' + esc(s.id) + '">Resume</button>' : '') + '</div>'; + } else { + note = '<div class="sessions-note">Your role can watch this session but not type into it, so you see a read-only scrollback that refreshes every few seconds.</div>'; + } + return note + '<pre class="sessions-log" id="sessions-log-' + esc(s.id) + '" tabindex="0" aria-label="Session scrollback">Loading scrollback…</pre>'; + } + function sessionsEnsurePane(s) { + const host = document.getElementById('sessions-panes'); + if (!host) return null; + let pane = document.getElementById('sessions-pane-' + s.id); + const mode = sessionsViewMode(_sessionsRole, s) === 'terminal' && _sessionsFrameFailed[s.id] ? 'scrollback' : sessionsViewMode(_sessionsRole, s); + if (pane && pane.getAttribute('data-mode') !== mode) { + // Mode changed (ended, resumed, role change): rebuild this pane only. + delete _sessionsFrames[s.id]; + pane.remove(); + pane = null; + } + if (!pane) { + pane = document.createElement('div'); + pane.id = 'sessions-pane-' + s.id; + pane.className = 'sessions-pane'; + pane.setAttribute('role', 'tabpanel'); + pane.setAttribute('aria-labelledby', 'sessions-tab-' + s.id); + pane.setAttribute('data-mode', mode); + pane.innerHTML = '<div class="sessions-pane-header"></div>' + sessionsPaneBodyHtml(s, mode); + host.appendChild(pane); + } + const head = pane.querySelector('.sessions-pane-header'); + if (head) head.innerHTML = sessionsHeaderHtml(s); + return pane; + } + function renderSessions() { + const panel = document.getElementById('sessions-panel'); + if (!panel) return; + refreshSectionCardShell('sessions-section'); + _sessionsCollapsed = Object.keys(_sessionsCollapsed).length ? _sessionsCollapsed : sessionsReadCollapsed(); + const sessions = _sessionsData.sessions || []; + setSectionSummary('sessions-section', sessionsSummary(sessions)); + let root = document.getElementById('sessions-root'); + if (!root) { + panel.innerHTML = '<div id="sessions-root" class="sessions-root"><div id="sessions-chrome"></div><div id="sessions-empty" class="sessions-note" hidden></div><div id="sessions-panes" class="sessions-panes"></div></div>'; + root = document.getElementById('sessions-root'); + } + // Re-render only the chrome so live iframes and a half-typed input survive polling. + const chrome = document.getElementById('sessions-chrome'); + const focusId = document.activeElement && document.activeElement.closest && document.activeElement.closest('#sessions-tabs') ? (document.activeElement.id || '') : ''; + const typing = document.activeElement && document.activeElement.closest && document.activeElement.closest('#sessions-new'); + if (!typing) chrome.innerHTML = sessionsToolbarHtml(); + if (focusId) { const f = document.getElementById(focusId); if (f) f.focus(); } + const empty = document.getElementById('sessions-empty'); + empty.hidden = sessions.length > 0 || _sessionsSelected !== SESSIONS_CAMPAIGNS_TAB; + empty.textContent = sessions.length ? '' : 'No sessions yet. Start one with “New session” to run a CLI agent here; everyone on this hive can watch or join it.'; + const liveIds = {}; + sessions.forEach(function(s) { liveIds[s.id] = true; sessionsEnsurePane(s); }); + Array.prototype.forEach.call(document.querySelectorAll('#sessions-panes .sessions-pane'), function(pane) { + const id = pane.id.replace('sessions-pane-', ''); + if (id !== SESSIONS_CAMPAIGNS_TAB && !liveIds[id]) { delete _sessionsFrames[id]; pane.remove(); } + }); + sessionsEnsureCampaignsPane(); + sessionsShowSelected(); + } + function sessionsSummary(sessions) { + const running = (sessions || []).filter(function(s) { return s.status === SESSIONS_STATUS_RUNNING; }).length; + return running + ' running · ' + (sessions || []).length + ' total'; + } + function sessionsEnsureCampaignsPane() { + const host = document.getElementById('sessions-panes'); + if (!host || document.getElementById('sessions-pane-' + SESSIONS_CAMPAIGNS_TAB)) return; + const pane = document.createElement('div'); + pane.id = 'sessions-pane-' + SESSIONS_CAMPAIGNS_TAB; + pane.className = 'sessions-pane'; + pane.setAttribute('role', 'tabpanel'); + pane.setAttribute('aria-labelledby', 'sessions-tab-' + SESSIONS_CAMPAIGNS_TAB); + pane.setAttribute('data-mode', 'campaigns'); + host.appendChild(pane); + } + function sessionsShowSelected() { + Array.prototype.forEach.call(document.querySelectorAll('#sessions-panes .sessions-pane'), function(pane) { + pane.hidden = pane.id !== 'sessions-pane-' + _sessionsSelected; + }); + if (_sessionsSelected === SESSIONS_CAMPAIGNS_TAB) { + sessionsStopLogPoll(); + const pane = document.getElementById('sessions-pane-' + SESSIONS_CAMPAIGNS_TAB); + if (pane && !pane.firstChild) { pane.innerHTML = renderCampaignsPanel(); loadCampaigns(); } + return; + } + const s = sessionsSelectedSession(); + if (!s) return; + const mode = sessionsViewMode(_sessionsRole, s); + if (mode === 'terminal' && !_sessionsFrameFailed[s.id]) { sessionsStopLogPoll(); sessionsAttachTerminal(s); } else sessionsStartLogPoll(); + } + + // ── Terminal iframe (lazy, kept alive when hidden) ── + async function sessionsHandoff(id) { + if (typeof renewTerminalAssertion === 'function') await renewTerminalAssertion(); + const resp = await fetch('/api/terminal/handoff', { method: 'POST', headers: sessionsHeaders(true), credentials: 'same-origin', cache: 'no-store', body: JSON.stringify({ session: id }) }); + const data = await resp.json().catch(function() { return {}; }); + if (!resp.ok || !data.code) { + const err = new Error(data.error || ('HTTP ' + resp.status)); + err.status = resp.status; + throw err; + } + return data.code; + } + async function sessionsAttachTerminal(s) { + if (_sessionsFrames[s.id]) return; + const host = document.querySelector('[data-terminal-host="' + s.id + '"]'); + if (!host) return; + _sessionsFrames[s.id] = 'pending'; + try { + const code = await sessionsHandoff(s.id); + const frame = document.createElement('iframe'); + frame.className = 'sessions-frame'; + frame.title = 'Terminal for session ' + (s.name || s.id); + frame.setAttribute('allow', 'clipboard-read; clipboard-write'); + frame.src = sessionsTerminalPath(s.id, code); + host.appendChild(frame); + _sessionsFrames[s.id] = frame; + sessionsStartAssertionRenew(); + } catch (e) { + delete _sessionsFrames[s.id]; + _sessionsFrameFailed[s.id] = true; + showToast('Could not open the terminal for ' + (s.name || s.id) + ': ' + e.message + '. Showing read-only scrollback instead.', 'error'); + const pane = document.getElementById('sessions-pane-' + s.id); + if (pane) { pane.remove(); } + renderSessions(); + } + } + function sessionsHasFrames() { + return Object.keys(_sessionsFrames).some(function(k) { return _sessionsFrames[k] && _sessionsFrames[k] !== 'pending'; }); + } + function sessionsStartAssertionRenew() { + if (_sessionsAssertionTimerOn) return; + _sessionsAssertionTimerOn = true; + dashboardSetInterval('sessions-assertion', async function() { + if (!sessionsHasFrames() || !sessionsIsEnabled()) { dashboardClearInterval('sessions-assertion'); _sessionsAssertionTimerOn = false; return; } + if (typeof renewTerminalAssertion === 'function') await renewTerminalAssertion(); + }, SESSIONS_ASSERTION_RENEW_MS); + } + + // ── Scrollback view (read role / ended sessions) ── + async function sessionsRefreshLog() { + const s = sessionsSelectedSession(); + if (!s || document.hidden) return; + const el = document.getElementById('sessions-log-' + s.id); + if (!el) return; + try { + const data = await sessionsApi('/' + encodeURIComponent(s.id) + '/log?lines=' + SESSIONS_LOG_LINES, 'GET'); + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight < el.clientHeight; + el.textContent = (data.lines || []).join('\n') || '(no output yet)'; + if (atBottom) el.scrollTop = el.scrollHeight; + } catch (e) { el.textContent = 'Could not load scrollback: ' + e.message; } + } + function sessionsStartLogPoll() { + sessionsRefreshLog(); + if (_sessionsLogTimerOn) return; + _sessionsLogTimerOn = true; + dashboardSetInterval('sessions-log', sessionsRefreshLog, SESSIONS_LOG_POLL_MS); + } + function sessionsStopLogPoll() { + if (!_sessionsLogTimerOn) return; + _sessionsLogTimerOn = false; + dashboardClearInterval('sessions-log'); + } + + // ── Actions (data-action handlers) ── + function sessionsSelect(id) { + if (!id || id === _sessionsSelected) return; + _sessionsSelected = id; + sessionsStoreSet(SESSIONS_SELECTED_KEY, id); + renderSessions(); + const tab = document.getElementById('sessions-tab-' + id); + if (tab && document.activeElement && document.activeElement.closest && document.activeElement.closest('#sessions-tabs')) tab.focus(); + } + function sessionsTabKey(id, ev) { + if (!ev) return; + const order = sessionsTabOrder(_sessionsData.sessions, _sessionsCollapsed, _sessionsSelected); + const next = sessionsNextTab(order, id, ev.key); + if (!next) return; + ev.preventDefault(); + sessionsSelect(next); + const tab = document.getElementById('sessions-tab-' + next); + if (tab) tab.focus(); + } + function sessionsToggleGroup(group) { + _sessionsCollapsed[group] = !_sessionsCollapsed[group]; + sessionsStoreSet(SESSIONS_COLLAPSED_KEY, JSON.stringify(_sessionsCollapsed)); + renderSessions(); + } + function sessionsToggleNew() { + _sessionsNewOpen = !_sessionsNewOpen; + renderSessions(); + const name = document.getElementById('sessions-new-name'); + if (name) name.focus(); + } + async function sessionsCreate() { + const val = function(id) { const el = document.getElementById(id); return el ? el.value.trim() : ''; }; + const body = { name: val('sessions-new-name'), group: val('sessions-new-group'), vendor: val('sessions-new-vendor') }; + if (!body.name) { showToast('Give the session a name.', 'error'); return; } + if (!body.vendor) { showToast('Pick a CLI vendor.', 'error'); return; } + try { + const data = await sessionsApi('', 'POST', body); + _sessionsNewOpen = false; + if (data.session) { _sessionsData.sessions.push(data.session); _sessionsSelected = data.session.id; sessionsStoreSet(SESSIONS_SELECTED_KEY, data.session.id); } + renderSessions(); + showToast('Started session ' + body.name + '.', 'success'); + loadSessions(); + } catch (e) { showToast('Could not start the session: ' + e.message, 'error'); } + } + async function sessionsPatch(id, patch, okMsg) { + try { + await sessionsApi('/' + encodeURIComponent(id), 'PATCH', patch); + showToast(okMsg, 'success'); + await loadSessions(); + } catch (e) { showToast('Could not update the session: ' + e.message, 'error'); } + } + async function sessionsRename(id) { + const s = (_sessionsData.sessions || []).find(function(x) { return x.id === id; }); + if (!s) return; + const name = await hivePrompt({ title: 'Rename session', label: 'New name', defaultValue: s.name || '', confirmLabel: 'Rename', validate: function(v) { return v ? true : 'Name cannot be empty.'; } }); + if (name && name !== s.name) sessionsPatch(id, { name: name }, 'Renamed to ' + name + '.'); + } + async function sessionsRegroup(id) { + const s = (_sessionsData.sessions || []).find(function(x) { return x.id === id; }); + if (!s) return; + const options = sessionsExistingGroups(_sessionsData.sessions).map(function(g) { return { value: g, label: '' }; }); + const group = await hivePrompt({ title: 'Change group', label: 'Group (leave empty for none)', defaultValue: s.group || '', confirmLabel: 'Move', options: options }); + if (group !== null && group !== undefined && group !== false && group !== (s.group || '')) sessionsPatch(id, { group: group }, group ? 'Moved to ' + group + '.' : 'Removed from its group.'); + } + async function sessionsClose(id) { + const s = (_sessionsData.sessions || []).find(function(x) { return x.id === id; }); + if (!s) return; + const ok = await hiveConfirm({ title: 'Close this session?', message: 'Closing "' + (s.name || s.id) + '" ends the CLI process for everyone attached and discards its scrollback.', confirmLabel: 'Close session', danger: true }); + if (!ok) return; + try { + await sessionsApi('/' + encodeURIComponent(id), 'DELETE'); + delete _sessionsFrames[id]; + _sessionsData.sessions = (_sessionsData.sessions || []).filter(function(x) { return x.id !== id; }); + if (_sessionsSelected === id) _sessionsSelected = sessionsPickSelected(_sessionsData.sessions, ''); + renderSessions(); + showToast('Session closed.', 'success'); + } catch (e) { showToast('Could not close the session: ' + e.message, 'error'); } + } + async function sessionsResume(id) { + try { + await sessionsApi('/' + encodeURIComponent(id) + '/resume', 'POST', {}); + showToast('Session resumed.', 'success'); + await loadSessions(); + } catch (e) { showToast('Could not resume the session: ' + e.message, 'error'); } + } + + // ── Input bar ── + function sessionsInputChanged(el) { + const bar = el && el.closest ? el.closest('.sessions-input') : null; + if (!bar) return; + const max = Number(_sessionsData.limits && _sessionsData.limits.max_input_bytes) || 0; + const counter = bar.querySelector('.sessions-input-count'); + if (!counter || !max) return; + const size = sessionsByteLength(el.value); + counter.textContent = sessionsFormatBytes(size) + ' / ' + sessionsFormatBytes(max); + counter.classList.toggle('over', size > max); + } + async function sessionsPostInput(id, text, submit) { + await sessionsApi('/' + encodeURIComponent(id) + '/input', 'POST', { text: text, submit: !!submit }); + } + async function sessionsSend(submit) { + const s = sessionsSelectedSession(); + if (!s) return; + const box = document.getElementById('sessions-input-text-' + s.id); + if (!box) return; + const check = sessionsCheckInput(box.value, _sessionsData.limits); + if (!check.ok) { showToast(check.error, 'error'); return; } + try { + await sessionsPostInput(s.id, box.value, String(submit) === '1' || submit === true); + box.value = ''; + sessionsInputChanged(box); + showToast('Sent to ' + (s.name || s.id) + '.', 'success'); + } catch (e) { showToast('Could not send: ' + e.message, 'error'); } + } + async function sessionsFilePicked(el) { + const s = sessionsSelectedSession(); + const file = el && el.files ? el.files[0] : null; + if (!s || !file) return; + const check = sessionsCheckUpload(file, _sessionsData.limits); + if (!check.ok) { showToast(check.error, 'error'); el.value = ''; return; } + try { + const form = new FormData(); + form.append('file', file); + const data = await sessionsApi('/' + encodeURIComponent(s.id) + '/upload', 'POST', form); + if (!data.path) throw new Error('the server returned no file path'); + await sessionsPostInput(s.id, data.path, false); + showToast('Uploaded ' + file.name + ' and typed its path into the session.', 'success'); + } catch (e) { showToast('Could not attach the file: ' + e.message, 'error'); } + el.value = ''; + } + // Registry slot: features add entries to the input bar's "More" menu. + function sessionsRegisterInputAction(action) { + if (!action || !action.id || !action.label || typeof action.run !== 'function') return false; + if (SESSIONS_INPUT_ACTIONS.some(function(a) { return a.id === action.id; })) return false; + SESSIONS_INPUT_ACTIONS.push(action); + if (_sessionsLoaded) renderSessions(); + return true; + } + function sessionsInputAction(id) { + const action = SESSIONS_INPUT_ACTIONS.find(function(a) { return a.id === id; }); + const s = sessionsSelectedSession(); + if (!action || !s) return; + const box = document.getElementById('sessions-input-text-' + s.id); + action.run({ session: s, role: _sessionsRole, textarea: box, send: function(text, submit) { return sessionsPostInput(s.id, text, submit); } }); + } + document.addEventListener('DOMContentLoaded', function() { + loadSessions(); + }); // ── Debug section ── const DEBUG_REFRESH_MS = 10000; const GH_RATE_CRITICAL_REMAINING_PCT = 10; diff --git a/src/pkg/dashboard/webstatic/static_index_test.go b/src/pkg/dashboard/webstatic/static_index_test.go index 9a1a9ad129..69dda455cc 100644 --- a/src/pkg/dashboard/webstatic/static_index_test.go +++ b/src/pkg/dashboard/webstatic/static_index_test.go @@ -420,7 +420,7 @@ func TestStaticDashboardDefaultLayoutOrder(t *testing.T) { } html := rec.Body.String() - defaultOrder := "'overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" + defaultOrder := "'overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','sessions-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" if !strings.Contains(html, defaultOrder) { t.Fatalf("dashboard default layout/reset order no longer matches sidebar nav; missing %q", defaultOrder) } From 636fcd9747f197a1cc8455bb8b422c3466fb2e80 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:45:12 +0000 Subject: [PATCH 185/223] fix: skip derived replica overlays so a stray <base>-N.yaml cannot crashloop the hive (#18140) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-18137-stray-replica-overlay.md | 1 + src/pkg/config/agent_overlay.go | 26 +++++ src/pkg/config/agent_replica_overlay_test.go | 108 ++++++++++++++++++ src/pkg/config/config.go | 38 ++++++ 4 files changed, 173 insertions(+) create mode 100644 changelog.d/fixed-18137-stray-replica-overlay.md create mode 100644 src/pkg/config/agent_replica_overlay_test.go diff --git a/changelog.d/fixed-18137-stray-replica-overlay.md b/changelog.d/fixed-18137-stray-replica-overlay.md new file mode 100644 index 0000000000..09f4900827 --- /dev/null +++ b/changelog.d/fixed-18137-stray-replica-overlay.md @@ -0,0 +1 @@ +- **A per-agent overlay for a derived replica no longer crash-loops the hive** (#18137). Dashboard saves on a replica such as `reviewer-2` used to write `reviewer-2.yaml` into `data.agents_dir` as if it were a base agent, and the next restart failed with `replicas:2 would create reviewer-2, but an agent with that name already exists` before the dashboard could bind. Replica edits are now written to the base agent's overlay, and a stray `<base>-N.yaml` already on disk is skipped at boot with an ERROR log naming the file, so the hive starts and the replica is re-derived from its base. diff --git a/src/pkg/config/agent_overlay.go b/src/pkg/config/agent_overlay.go index 3d5f32d944..f8b56b700e 100644 --- a/src/pkg/config/agent_overlay.go +++ b/src/pkg/config/agent_overlay.go @@ -273,10 +273,36 @@ func (c *Config) ClearAgentRemoved(name string) bool { // only defer the failure to a restart. The gateway-aware half of the gate // needs the governor config; callers that have it should also run // Config.ValidateAgentLaunchCmds before calling this. +// +// A derived replica (ReplicaOf set) is never written under its own name: +// ReplicaOf is not persisted, so reviewer-2.yaml would load as a base agent +// that collides with reviewer.replicas on the next boot (#18137). Its edit is +// routed to the base agent's overlay instead, with the replica's identity +// (id, beads_dir) restored to the base's. A review head cannot be routed — +// its backend and model differ from the base by design — so it is refused. func SaveAgentFile(dir, name string, agent AgentConfig) error { if strings.Contains(name, "..") || strings.Contains(name, "/") || strings.Contains(name, "\\") { return fmt.Errorf("invalid agent name %q", name) } + if agent.ReplicaOf != "" { + if agent.ReviewHead != "" { + return fmt.Errorf("refusing to save agent overlay %s: it is review head %s of %s; edit the base agent", name, agent.ReviewHead, agent.ReplicaOf) + } + derived := name + name = agent.ReplicaOf + if strings.Contains(name, "..") || strings.Contains(name, "/") || strings.Contains(name, "\\") { + return fmt.Errorf("invalid agent name %q", name) + } + if agent.ID == "" || agent.ID == derived { + agent.ID = name + } + if agent.BeadsDir == fmt.Sprintf("/data/beads/%s", derived) { + agent.BeadsDir = fmt.Sprintf("/data/beads/%s", name) + } + agent.ReplicaOf = "" + agent.ReplicaIndex = 0 + agent.ReplicaCount = 0 + } if err := (GovernorConfig{}).ValidateLaunchCmdBackend(agent.Backend, agent.LaunchCmd); err != nil { return fmt.Errorf("refusing to save agent overlay %s: %w", name, err) } diff --git a/src/pkg/config/agent_replica_overlay_test.go b/src/pkg/config/agent_replica_overlay_test.go new file mode 100644 index 0000000000..43c982d46f --- /dev/null +++ b/src/pkg/config/agent_replica_overlay_test.go @@ -0,0 +1,108 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// #18137: an overlay-sourced reviewer-2 (a serialized derived replica, so +// ReplicaOf is empty) next to reviewer.replicas: 2 must not fail the load; the +// stray is dropped and exactly two reviewers run. +func TestExpandAgentReplicasDropsStrayReplicaOverlay(t *testing.T) { + stray := AgentConfig{ID: "reviewer-2", Backend: "copilot", Replicas: 2, DisplayName: "reviewer"} + stray.SetLoadState("reviewer-2", "/data/agent-configs/reviewer-2.yaml") + cfg := &Config{AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "reviewer": {ID: "reviewer", Backend: "copilot", Enabled: true, Replicas: 2}, + "reviewer-2": stray, + }}} + + if err := cfg.ExpandAgentReplicas(); err != nil { + t.Fatalf("ExpandAgentReplicas() error = %v, want nil (a stray replica overlay must not brick the hive)", err) + } + var reviewers []string + for name := range cfg.Agents { + if strings.HasPrefix(name, "reviewer") { + reviewers = append(reviewers, name) + } + } + if len(reviewers) != 2 { + t.Fatalf("reviewer agents = %v, want exactly reviewer and reviewer-2", reviewers) + } + if got := cfg.Agents["reviewer-2"]; got.ReplicaOf != "reviewer" || got.ReplicaIndex != 2 { + t.Fatalf("reviewer-2 = %+v, want the replica derived from reviewer", got) + } +} + +// A colliding agent declared in the main config (no overlay file) is still +// rejected: only a stale overlay copy is safe to drop. +func TestExpandAgentReplicasStillRejectsDeclaredCollision(t *testing.T) { + cfg := &Config{AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "reviewer": {Replicas: 2}, + "reviewer-2": {Replicas: 1}, + }}} + if err := cfg.ExpandAgentReplicas(); err == nil || !strings.Contains(err.Error(), "reviewer-2") { + t.Fatalf("collision error = %v, want an error naming reviewer-2", err) + } +} + +// The incident end to end: a stray worker-2.yaml beside a worker overlay with +// replicas: 2 must load. +func TestLoadWithOverrides_StrayReplicaOverlayDoesNotFailLoad(t *testing.T) { + configPath, _ := writeOverlayConfig(t, map[string]string{ + "worker": "id: worker\nbackend: claude\nenabled: true\nreplicas: 2\n", + "worker-2": "id: worker-2\nbackend: claude\nenabled: true\nreplicas: 2\ndisplay_name: worker\n", + }) + cfg, err := LoadWithOverrides(configPath, "-") + if err != nil { + t.Fatalf("LoadWithOverrides() error = %v, want nil", err) + } + if got := cfg.Agents["worker-2"]; got.ReplicaOf != "worker" { + t.Fatalf("worker-2.ReplicaOf = %q, want worker", got.ReplicaOf) + } +} + +// The writer half of #18137: saving a derived replica writes the base agent's +// overlay, never <base>-N.yaml. +func TestSaveAgentFileRoutesReplicaToBase(t *testing.T) { + dir := t.TempDir() + replica := AgentConfig{ + ID: "reviewer-2", + Backend: "copilot", + Model: "gpt-5", + BeadsDir: "/data/beads/reviewer-2", + Replicas: 2, + ReplicaOf: "reviewer", + ReplicaIndex: 2, + ReplicaCount: 2, + } + if err := SaveAgentFile(dir, "reviewer-2", replica); err != nil { + t.Fatalf("SaveAgentFile() error = %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "reviewer-2.yaml")); !os.IsNotExist(err) { + t.Fatalf("reviewer-2.yaml exists (stat err %v); a derived replica must never get its own overlay", err) + } + agents, err := LoadAgentOverrides(dir) + if err != nil { + t.Fatalf("LoadAgentOverrides() error = %v", err) + } + base, ok := agents["reviewer"] + if !ok { + t.Fatalf("reviewer.yaml not written; overlays = %v", agents) + } + if base.ID != "reviewer" || base.BeadsDir != "/data/beads/reviewer" || base.Model != "gpt-5" || base.Replicas != 2 { + t.Fatalf("routed overlay = %+v, want the base identity with the replica's edit", base) + } +} + +func TestSaveAgentFileRefusesReviewHead(t *testing.T) { + dir := t.TempDir() + head := AgentConfig{ID: "reviewer-gpt", Backend: "codex", ReplicaOf: "reviewer", ReviewHead: "gpt"} + if err := SaveAgentFile(dir, "reviewer-gpt", head); err == nil { + t.Fatal("SaveAgentFile() error = nil, want a refusal for a review head") + } + if entries, _ := os.ReadDir(dir); len(entries) != 0 { + t.Fatalf("overlay dir has %d entries, want none", len(entries)) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 4f60d58d69..a38e64c09d 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -7466,6 +7466,7 @@ func (c *Config) ExpandAgentReplicas() error { } return normalizeReplicaCount(agent.Replicas) } + c.dropStrayReplicaOverlays(baseAgents, effectiveReplicas) derivedNames := map[string]string{} for name, agent := range baseAgents { replicas := normalizeReplicaCount(agent.Replicas) @@ -7528,6 +7529,43 @@ func (c *Config) ExpandAgentReplicas() error { return nil } +// dropStrayReplicaOverlays removes from baseAgents (and c.Agents) every +// per-agent overlay whose name is a replica that another base agent's +// replicas setting derives: reviewer-2.yaml next to reviewer.replicas: 2 +// (#18137). Such a file is a serialized derived replica — ReplicaOf is not +// persisted, so it loads as a base agent — and the collision used to fail +// ExpandAgentReplicas, which exits the process before the dashboard binds and +// crash-loops the hive with no supported way back in (same shape as #6024). +// +// Only overlay-sourced entries are dropped; a colliding agent declared in the +// main config is still an error, because nothing in it can be a stale copy. +// Dropping the file changes nothing that runs: the replica it names is +// re-derived from the base agent below. Every drop is logged at ERROR naming +// the file so the operator can delete it. +func (c *Config) dropStrayReplicaOverlays(baseAgents map[string]AgentConfig, effectiveReplicas func(string, AgentConfig) int) { + for name, agent := range baseAgents { + if n := normalizeReplicaCount(agent.Replicas); n < 1 || n > MaxAgentReplicas { + continue // reported by the caller's range check + } + replicas := effectiveReplicas(name, agent) + for i := 2; i <= replicas; i++ { + derived := replicaAgentName(name, i) + stray, ok := baseAgents[derived] + if !ok || stray.ReplicaOf != "" || stray.SourceFile() == "" { + continue + } + slog.Default().Error("config: ignored per-agent overlay file for a derived replica - the replica is re-derived from its base agent; delete the file (#18137)", + "hive_id", c.HiveID, + "agent", derived, + "replica_of", name, + "file", stray.SourceFile(), + ) + delete(baseAgents, derived) + delete(c.Agents, derived) + } + } +} + // MarshalYAML persists only declared agents. Runtime-derived replicas are // re-created by ExpandAgentReplicas on the next load so they never collide with // their base agent's replicas setting after a save. From ec4cdf99420d114132e57f6379e4e9973cf07986 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:48:45 +0000 Subject: [PATCH 186/223] =?UTF-8?q?=E2=9C=A8=20config:=20add=20project.ref?= =?UTF-8?q?erence=5Frepos=20with=20tier=20validation=20and=20ReferenceRepo?= =?UTF-8?q?Tier=20accessor=20(#18027)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ config: add project.reference_repos with tier validation and ReferenceRepoTier accessor Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * ✨ config: cover blank/nil guards of the reference repo accessors The config package sat at 89.9% against its 90% floor; the new accessor's blank-slug guard and the nil-receiver guards were never exercised. Signed-off-by: adjudicator-2 <280983584+hivecommons-hive[bot]@users.noreply.github.com> * ♻️ config: drop ValidateReferenceRepos wrapper, dedupe tier label and lookup Hive-Fix-Attempt: 5/6 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: adjudicator-2 <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: adjudicator-3 <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: adjudicator <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: adjudicator-4 <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- .../added-reference-repo-tier-accessor.md | 1 + src/pkg/config/reference_repos.go | 52 +++++++++++++++---- src/pkg/config/reference_repos_test.go | 51 ++++++++++++++++++ 3 files changed, 93 insertions(+), 11 deletions(-) create mode 100644 changelog.d/added-reference-repo-tier-accessor.md diff --git a/changelog.d/added-reference-repo-tier-accessor.md b/changelog.d/added-reference-repo-tier-accessor.md new file mode 100644 index 0000000000..5b7891cedd --- /dev/null +++ b/changelog.d/added-reference-repo-tier-accessor.md @@ -0,0 +1 @@ +- `config.ProjectConfig` gains a `ReferenceRepoTier(slug)` accessor (and `ReferenceRepo.Tier()`) returning a reference repo's write tier, so the proxy, kick renderer and dashboard share one source of truth (#18018). diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go index f67a976c34..bb5093ebeb 100644 --- a/src/pkg/config/reference_repos.go +++ b/src/pkg/config/reference_repos.go @@ -96,17 +96,7 @@ func (r ReferenceRepo) hasWrite(kind string) bool { // WriteLabel is the human-facing tier: "read-only", "issues", "prs" or // "issues+prs". The dashboard chip and the kick both print it. -func (r ReferenceRepo) WriteLabel() string { - switch { - case r.WritesIssues() && r.WritesPRs(): - return "issues+prs" - case r.WritesIssues(): - return "issues" - case r.WritesPRs(): - return "prs" - } - return "read-only" -} +func (r ReferenceRepo) WriteLabel() string { return r.Tier().String() } // Owner returns the owner half of Repo ("" when malformed). func (r ReferenceRepo) Owner() string { @@ -229,3 +219,43 @@ func (c *Config) ReferenceRepoSlugs() []string { sort.Strings(out) return out } + +// ReferenceRepoTier is the access tier of a reference repo as a pair of +// flags, for callers that branch on the opt-ins rather than print them. +type ReferenceRepoTier struct { + Issues bool + PRs bool +} + +// ReferenceTierReadOnly is the default tier: no writes. +var ReferenceTierReadOnly = ReferenceRepoTier{} + +// String yields the same label as ReferenceRepo.WriteLabel: "read-only", +// "issues", "prs" or "issues+prs". +func (t ReferenceRepoTier) String() string { + switch { + case t.Issues && t.PRs: + return "issues+prs" + case t.Issues: + return ReferenceWriteIssues + case t.PRs: + return ReferenceWritePRs + } + return "read-only" +} + +// Tier returns the entry's write opt-ins as a ReferenceRepoTier. +func (r ReferenceRepo) Tier() ReferenceRepoTier { + return ReferenceRepoTier{Issues: r.WritesIssues(), PRs: r.WritesPRs()} +} + +// ReferenceRepoTier returns the tier of the reference repo matching slug +// ("owner/name", case-insensitive) and false when slug is not a reference +// repo. +func (p ProjectConfig) ReferenceRepoTier(slug string) (ReferenceRepoTier, bool) { + r, ok := (&Config{Project: p}).ReferenceRepoFor(slug) + if !ok { + return ReferenceTierReadOnly, false + } + return r.Tier(), true +} diff --git a/src/pkg/config/reference_repos_test.go b/src/pkg/config/reference_repos_test.go index f0973865ec..558e4674b4 100644 --- a/src/pkg/config/reference_repos_test.go +++ b/src/pkg/config/reference_repos_test.go @@ -99,3 +99,54 @@ func TestReferenceReposValidationRunsFromValidate(t *testing.T) { t.Fatalf("Validate should surface the reference_repos error, got %v", err) } } + +func TestReferenceRepoTierAccessor(t *testing.T) { + c := referenceRepoTestConfig(t, ` + - hivecommons/docs + - repo: hivecommons/hive + write: [issues, prs] + - repo: hivecommons/site + write: [prs] +`) + tier, ok := c.Project.ReferenceRepoTier("HiveCommons/Docs") + if !ok || tier != ReferenceTierReadOnly || tier.String() != "read-only" { + t.Fatalf("docs: tier=%v ok=%v", tier, ok) + } + tier, ok = c.Project.ReferenceRepoTier("hivecommons/hive") + if !ok || !tier.Issues || !tier.PRs || tier.String() != "issues+prs" { + t.Fatalf("hive: tier=%v ok=%v", tier, ok) + } + tier, _ = c.Project.ReferenceRepoTier("hivecommons/site") + if tier.Issues || !tier.PRs || tier.String() != "prs" { + t.Fatalf("site: tier=%v", tier) + } + if _, ok := c.Project.ReferenceRepoTier("hivecommons/nope"); ok { + t.Fatal("unknown slug must report ok=false") + } + if _, ok := c.Project.ReferenceRepoTier("cncf/techdocs"); ok { + t.Fatal("authorized repo must not report a reference tier") + } +} + +func TestReferenceRepoTierEmptyAndNilGuards(t *testing.T) { + c := referenceRepoTestConfig(t, " - hivecommons/docs\n") + if tier, ok := c.Project.ReferenceRepoTier(" "); ok || tier != ReferenceTierReadOnly { + t.Fatalf("blank slug: tier=%v ok=%v", tier, ok) + } + var nilCfg *Config + if _, ok := nilCfg.ReferenceRepoFor("hivecommons/docs"); ok { + t.Fatal("nil config must not match") + } + if _, ok := c.ReferenceRepoFor(""); ok { + t.Fatal("blank slug must not match") + } + if got := nilCfg.ReferenceReposWithWrite(); got != nil { + t.Fatalf("nil config: want nil, got %v", got) + } + if got := nilCfg.ReferenceRepoSlugs(); got != nil { + t.Fatalf("nil config: want nil, got %v", got) + } + if got := c.ReferenceRepoSlugs(); len(got) != 1 || got[0] != "hivecommons/docs" { + t.Fatalf("slugs: %v", got) + } +} From 47ede19072e63356acbc8c63a1c3455a3b6cc871 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 23:51:32 +0000 Subject: [PATCH 187/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.248.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.248.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-18115-proxy-uid-identity-spoof.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-18115-proxy-uid-identity-spoof.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e6903dcf80..2803720f9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.248.1) + +### Fixed + +- The inference translator no longer lets a caller at the shared proxy UID (`dev`, 1001 on v5 pods, where the hive process and every agent without a per-agent UID run) claim the `sk-hive-<name>` key of an agent that has its own UID: such an agent always connects from that UID, so the claim is refused with `reason=proxy_uid_claims_isolated_agent` and the victim's route and upstream credential are never used. Names with no per-agent UID are still accepted from the proxy UID, root stays trusted, and `HIVE_PROXY_ADVISORY_OK` still accepts self-asserted identity only where the socket proves nothing, not when it proves the claim false (#18115). + ## 2026-11-13 (v5.248.0) ### Added diff --git a/changelog.d/fixed-18115-proxy-uid-identity-spoof.md b/changelog.d/fixed-18115-proxy-uid-identity-spoof.md deleted file mode 100644 index 6f6dec0214..0000000000 --- a/changelog.d/fixed-18115-proxy-uid-identity-spoof.md +++ /dev/null @@ -1 +0,0 @@ -- The inference translator no longer lets a caller at the shared proxy UID (`dev`, 1001 on v5 pods, where the hive process and every agent without a per-agent UID run) claim the `sk-hive-<name>` key of an agent that has its own UID: such an agent always connects from that UID, so the claim is refused with `reason=proxy_uid_claims_isolated_agent` and the victim's route and upstream credential are never used. Names with no per-agent UID are still accepted from the proxy UID, root stays trusted, and `HIVE_PROXY_ADVISORY_OK` still accepts self-asserted identity only where the socket proves nothing, not when it proves the claim false (#18115). From 7f9abf2439b5d64b1b8d064e783980c1434d203a Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 18:55:44 -0500 Subject: [PATCH 188/223] =?UTF-8?q?=F0=9F=90=9B=20fix(stage-engine):=20arc?= =?UTF-8?q?hitecture=20discussion=20and=20[RFC]=20titles=20derive=20018,?= =?UTF-8?q?=20not=20010=20(#18149)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(stage-engine): architecture discussion and [RFC] titles derive 018, not 010 RFCs labelled `architecture discussion` (hivecommons/hive#18068, #18069) derived stage/010-actionable and showed as idle agent work because discussionLabels was the hard-coded pair proposal / kind/design. - add `architecture discussion` to discussionLabels; the most specific label is reported first - treat an `[RFC]` / `RFC:` title opener (case-insensitive, after any leading emoji) as a discussion signal when no discussion label is present, reason `open for discussion: rfc title`; IssueStageInput and stageengine.Facts carry the title for it - a live claim or covering PR still stages 020 / 030 / 040 - tests: signal resolver, IssueStageFor, deriver table, Facts parity and goldens for the labelled, title-only and claimed shapes - docs: stage table, plan, ADR-0025 and governor-thresholds name the new label and the title rule Fixes #18141 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 fix(stage-engine): golden for a claimed [RFC] issue must not wear Epic (umbrella claims are not staged) Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...ed-18141-stage-engine-discussion-labels.md | 1 + src/docs/adr/0025-central-stage-engine.md | 8 +- src/docs/governor-thresholds.md | 2 +- .../stage-labels.md | 12 +-- src/docs/stage-engine-plan.md | 10 ++- src/pkg/github/issue_stage_facts.go | 5 +- src/pkg/github/issue_stage_facts_test.go | 5 ++ src/pkg/github/issue_stage_goldens_test.go | 38 +++++++--- src/pkg/github/issue_stage_labels.go | 73 +++++++++++++++---- src/pkg/github/issue_stage_labels_test.go | 65 +++++++++++++++++ .../issue_stage_goldens/regressions.json | 5 +- src/pkg/stageengine/derive_issue.go | 5 +- src/pkg/stageengine/derive_issue_test.go | 18 +++++ src/pkg/stageengine/facts.go | 34 +++++---- 14 files changed, 225 insertions(+), 56 deletions(-) create mode 100644 changelog.d/fixed-18141-stage-engine-discussion-labels.md diff --git a/changelog.d/fixed-18141-stage-engine-discussion-labels.md b/changelog.d/fixed-18141-stage-engine-discussion-labels.md new file mode 100644 index 0000000000..ecc61bff51 --- /dev/null +++ b/changelog.d/fixed-18141-stage-engine-discussion-labels.md @@ -0,0 +1 @@ +- The stage engine now treats `architecture discussion` as a discussion label beside `proposal` and `kind/design`, and reads an `[RFC]` / `RFC:` title opener (after any leading emoji) as a discussion signal when no discussion label is present, so an unclaimed RFC derives `stage/018-open-for-discussion` (reason `open for discussion: architecture discussion` or `open for discussion: rfc title`) instead of showing as idle agent work at 010. A live claim or covering PR still stages 020 / 030 / 040 as before (#18141). diff --git a/src/docs/adr/0025-central-stage-engine.md b/src/docs/adr/0025-central-stage-engine.md index 6735591478..8bdc416309 100644 --- a/src/docs/adr/0025-central-stage-engine.md +++ b/src/docs/adr/0025-central-stage-engine.md @@ -428,9 +428,11 @@ Leaving 017 for 010 or closed is never a regression. **Open for discussion (`stage/018-open-for-discussion`, #17150).** Added after the epic: an issue-only stage between 017 and 020 with no dwell. Unlike 017 it is derived, not written by a gate: `DeriveIssue` returns it -for an open issue labelled `proposal` or `kind/design` with no live claim -(`IssueFacts.DiscussionLabel`, resolved in `pkg/github`), reason -`open for discussion: <label>`. It is not a suppress: the claim gate is +for an open issue labelled `proposal`, `kind/design` or `architecture +discussion` with no live claim, or an unlabelled one titled `[RFC]` / +`RFC:` (`IssueFacts.DiscussionLabel`, resolved in `pkg/github` from the +labels and the title, #18141), reason `open for discussion: <label>` or +`open for discussion: rfc title`. It is not a suppress: the claim gate is unchanged and a claim stages 020/030. `040 → 018` under `hive/partially-addressed` is progress; leaving 018 is never a regression. diff --git a/src/docs/governor-thresholds.md b/src/docs/governor-thresholds.md index a0bf3cbe10..7aecdbed28 100644 --- a/src/docs/governor-thresholds.md +++ b/src/docs/governor-thresholds.md @@ -2,7 +2,7 @@ The governor picks a mode — **idle → quiet → busy → surge** — by comparing queue depth (actionable issues + open PRs, summed across every repo the hive watches) against three thresholds. The mode then selects each agent's kick cadence, so the thresholds are what decide how hard the hive works. -Queue depth counts the enumerated work queue, not stage labels: an issue at `stage/018-open-for-discussion` (an unclaimed `proposal` / `kind/design` issue, [#17150](https://github.com/hivecommons/hive/issues/17150)) is still an open, claimable issue and still counts toward the actionable issue depth, exactly as it did when it read `stage/010-actionable`. The stage only changes how the activity monitor labels it. +Queue depth counts the enumerated work queue, not stage labels: an issue at `stage/018-open-for-discussion` (an unclaimed `proposal` / `kind/design` / `architecture discussion` or `[RFC]`-titled issue, [#17150](https://github.com/hivecommons/hive/issues/17150)) is still an open, claimable issue and still counts toward the actionable issue depth, exactly as it did when it read `stage/010-actionable`. The stage only changes how the activity monitor labels it. ## The problem the scaling solves diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index 2dea077700..1f4d4cb444 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -95,7 +95,7 @@ referenced it) a lone `stage/050-review-pending` | `stage/010-actionable` | issue | triaged; eligible for the hive to pick up. A human-filed issue without `needs-triage` counts as triaged by its trusted reporter | 24h | | `stage/015-triage-needed` | issue | actionable, but no maintainer has looked at it yet. Derived instead of 010 only when the reporter-trust gate is holding the issue: it carries the awaiting label `needs-triage` (`reporter_trust.awaiting_label` default) or sits in the #9665 gate's `reporter_triage` bucket. Who filed it is never consulted — issues from hive agents and from trusted humans are triaged by construction and stay 010. A **display and tracking state, not a gate**: the scanner and relay claim a 015 issue exactly as a 010 one (the reporter-trust gate withholds `needs-triage` issues separately, and that is unchanged), and 015 lights the activity monitor's *Actionable* column as "Triage needed". `015 → 010` (a maintainer triages) is never a regression; `010 → 015` is a reclassification. Because 015 is not a gate, a claim settles triage: whenever the sweep (or an out-of-cycle claim fact) writes a stage of 020 or later — 020, 030, 040 or a PR-mirrored stage — it removes `needs-triage` in the same label write, and an open issue already at 020+ still wearing it is reconciled on the next sweep; the label survives at 010, 017, 018 and 045, which the table lets fall back to 015 (hivecommons/hive#14586). The Open column is untouched: an issue with no stage stays "Open" | 24h | | `stage/017-possible-existing` | issue | **Possibly exists · needs author.** The prior-art gate found something that looks like the request already (knowledge bead, setting, merged PR, closed issue, doc or code) and parked the issue with the findings; the author confirms the gap (`/hive decision proceed — <gap>`) or closes it (`/hive decision exists`, label `already-exists`). The label itself is the park signal and an issue-level hard suppress, so no agent claims it; clanker-filed parks auto-close after `runs.triage.prior_art.bot_close_ttl`; hive-filed issues never wear it ([#13575](https://github.com/hivecommons/hive/issues/13575)) | none | -| `stage/018-open-for-discussion` | issue | **Open for discussion.** An open issue labelled `proposal` or `kind/design` with no live claim: what is left on it is a discussion, not agent work, so it is derived here instead of 010 / 015 and the activity monitor's *Actionable* column reads "Open for discussion", the caption naming the label. The stage reason is `open for discussion: <label>`. **Not a gate**: the claim gate does not refuse these labels, so a person's `/claim`, the contribute relay or an agent claim stages it 020 / 030 as on any issue; parks, holds, `blocked` and a covering PR outrank it as they do 010. Removing the label returns the issue to plain 010 derivation (never a regression), and a covering `Refs #N` PR that merges and leaves `hive/partially-addressed` lands it back here as progress ([#17150](https://github.com/hivecommons/hive/issues/17150)) | none | +| `stage/018-open-for-discussion` | issue | **Open for discussion.** An open issue labelled `proposal`, `kind/design` or `architecture discussion` with no live claim, or an unlabelled one whose title opens with `[RFC]` or `RFC:` (case-insensitive, after any leading emoji): what is left on it is a discussion, not agent work, so it is derived here instead of 010 / 015 and the activity monitor's *Actionable* column reads "Open for discussion", the caption naming the label. The stage reason is `open for discussion: <label>`, or `open for discussion: rfc title` when only the title marks it ([#18141](https://github.com/hivecommons/hive/issues/18141)). **Not a gate**: the claim gate does not refuse these labels, so a person's `/claim`, the contribute relay or an agent claim stages it 020 / 030 as on any issue; parks, holds, `blocked` and a covering PR outrank it as they do 010. Removing the label returns the issue to plain 010 derivation (never a regression), and a covering `Refs #N` PR that merges and leaves `hive/partially-addressed` lands it back here as progress ([#17150](https://github.com/hivecommons/hive/issues/17150)) | none | | `stage/020-claimed` | issue | a live claim by anyone — a human, a relay contributor or a hive agent's `hive-open-issue claim` / start mark — with no PR evidence yet (replaces `claimed`, `hive/in-progress`) | 2h | | `stage/030-pr-in-progress` | issue | the claimant has a PR under way: its `hive-open-pr --issues` request named the issue or its start marker named a branch / PR (`pr_started=true` on the `hive:claim` marker). A claim or start mark without that evidence — the scanner's bare `hive-open-issue claim` included — stays at 020 ([#13069](https://github.com/hivecommons/hive/issues/13069)). If the in-progress evidence includes an open Hive-authored PR, the issue-stage lane applies the same PR-link safeguard described for 040: default-branch PRs that fully address the issue get a `Fixes #N` trailer when missing, while partial or non-default-branch PRs get an issue stage-entry comment linking the PR and explaining GitHub's icon limitation. If the claiming PR has already merged, the issue leaves 030: full-fix claims close it with one `<!-- hive:closed-by-merged-pr pr=N -->` comment (checked before posting, so a close that fails after the comment is retried without a duplicate), while `Refs` / `Part of` claims move back to 010 with a partial-landed note. | 90m | | `stage/040-covered-by-pr` | issue | an open PR covers it (replaces `🐝 covered by PR`), or — when the hive's PR only *references* the issue without a closing keyword (`Refs #N`) — "an open PR addresses part of it": the issue is still parked here but its stage reason and tracker caption say the remainder stays open when the PR merges ([#15008](https://github.com/hivecommons/hive/issues/15008)). Coverage is derived only from a closing reference in the PR body or commits, a hive claim-ledger entry, or an explicit hive marker — a bare cross-reference (an incident deferral comment, `see also`, a backlog link) never puts an issue here; the issue stays here while the PR is open — the PR's own stage is named in the issue's stage reason, never written as its label ([#12451](https://github.com/hivecommons/hive/issues/12451)). For Hive-authored PRs on the repository default branch, the stage and claim-label sweeps append `Fixes #N` to the PR body when the PR appears to fully address the issue but lacks a closing keyword, so GitHub can show the Development sidebar link and issue-list PR icon. They also retrofit older Hive-authored default-branch PRs whose only claim line is `Refs #N` by rewriting that one keyword to `Fixes #N` once and stamping `<!-- hive:issue-link-fixed -->`; human-authored PR bodies are never edited. `Part of` / leave-open partial work and non-default-branch PRs get one issue stage-entry comment with the PR link and GitHub limitation instead. Merged full-fix claims close the issue; merged reference claims remove stale 040/030 and return the issue to 010 with a partial-landed note. | n/a (the PR carries health) | @@ -122,7 +122,7 @@ stateDiagram-v2 direction LR [*] --> s010: issue triaged [*] --> s015: issue from an untrusted reporter (needs-triage / reporter_triage) - [*] --> s018: proposal / kind/design issue with no live claim + [*] --> s018: proposal / kind/design / architecture discussion issue, or an RFC-titled one, with no live claim s010: 010 actionable s015: 015 triage-needed s018: 018 open-for-discussion @@ -147,10 +147,10 @@ stateDiagram-v2 s015 --> s010: maintainer triages: triage/accepted, hive/verified-open or approved-direction added, or needs-triage removed (not a regression) s015 --> s020: scanner / relay claim (015 is not a gate; needs-triage removed in the same write) s015 --> s045: needs-human / park label (agent escalates) - s010 --> s018: proposal / kind/design label added to an unclaimed issue (reclassification) - s018 --> s010: proposal / kind/design label removed (not a regression) - s015 --> s018: proposal / kind/design label added to an untriaged issue (reclassification) - s018 --> s015: proposal / kind/design label removed on an untriaged issue (not a regression) + s010 --> s018: discussion label added to an unclaimed issue, or it is retitled as an RFC (reclassification) + s018 --> s010: discussion label and RFC title removed (not a regression) + s015 --> s018: discussion label added to an untriaged issue (reclassification) + s018 --> s015: discussion label and RFC title removed on an untriaged issue (not a regression) s018 --> s020: a person, relay contributor or agent claims it (018 is not a gate) s018 --> s045: needs-human / park label s045 --> s018: /hive approve or /hive decision on a proposal diff --git a/src/docs/stage-engine-plan.md b/src/docs/stage-engine-plan.md index b8a37ba8e0..deb4fec93b 100644 --- a/src/docs/stage-engine-plan.md +++ b/src/docs/stage-engine-plan.md @@ -271,9 +271,13 @@ Shipped as three PRs (hivecommons/hive#13011): still live) and `030 → 020` (PR evidence lapsed, claim held). - **018 open for discussion** ([#17150](https://github.com/hivecommons/hive/issues/17150)): - an open issue labelled `proposal` or `kind/design` with no live claim - derives `stage/018-open-for-discussion` instead of 010 / 015. - `pkg/github` resolves the label into `IssueFacts.DiscussionLabel`; the + an open issue labelled `proposal`, `kind/design` or `architecture + discussion` with no live claim derives `stage/018-open-for-discussion` + instead of 010 / 015, as does an unlabelled issue whose title opens with + `[RFC]` / `RFC:` (reason `open for discussion: rfc title`, + [#18141](https://github.com/hivecommons/hive/issues/18141)). + `pkg/github` resolves the label or title into + `IssueFacts.DiscussionLabel`; the deriver applies it wherever the issue would otherwise sit unclaimed (the actionable / reporter-triage buckets, and an umbrella whose claim is not staged), and names the label in the reason (`open for discussion: diff --git a/src/pkg/github/issue_stage_facts.go b/src/pkg/github/issue_stage_facts.go index f4b80c2308..d8c948138c 100644 --- a/src/pkg/github/issue_stage_facts.go +++ b/src/pkg/github/issue_stage_facts.go @@ -29,6 +29,7 @@ func issueFactsFrom(in IssueStageInput, prStage func(repo string, number int) st f := stageengine.Facts{ Ref: stageengine.Ref{Owner: owner, Repo: repo, Number: in.Number, Kind: stagelabels.IssueItem}, Labels: append([]string(nil), in.Labels...), + Title: in.Title, Open: true, Author: stageengine.AuthorFacts{Kind: in.AuthorKind}, Issue: &stageengine.IssueFacts{ @@ -103,7 +104,7 @@ func resolveIssueLabelFacts(in IssueStageInput, f *stageengine.Facts) { is.ApprovedDirectionLabel = HumanAckLabel } is.UmbrellaLabel = issueUmbrellaLabel(in.Labels, in.SpekUmbrella) - is.DiscussionLabel = issueDiscussionLabel(in.Labels) + is.DiscussionLabel = issueDiscussionSignal(in.Labels, in.Title) is.BlockedReason = "" if hasBlockedLabel(in.Labels) { is.BlockedReason = blockedStageReason(in) @@ -129,7 +130,7 @@ var issueBlockerLookup = func(repo string, number int) (itemblocker.Blocker, boo // (a FactSink submission enriched with a fresh read) so the same label // resolution applies to it as to an enumerated issue. func issueInputFrom(f stageengine.Facts) IssueStageInput { - in := IssueStageInput{Repo: f.Ref.FullRepo(), Number: f.Ref.Number, Labels: f.Labels, AuthorKind: f.Author.Kind} + in := IssueStageInput{Repo: f.Ref.FullRepo(), Number: f.Ref.Number, Labels: f.Labels, Title: f.Title, AuthorKind: f.Author.Kind} if f.Issue != nil { if f.Issue.Bucket != stageengine.BucketActionable { in.Bucket = string(f.Issue.Bucket) diff --git a/src/pkg/github/issue_stage_facts_test.go b/src/pkg/github/issue_stage_facts_test.go index b221bd9e65..88188133be 100644 --- a/src/pkg/github/issue_stage_facts_test.go +++ b/src/pkg/github/issue_stage_facts_test.go @@ -41,6 +41,11 @@ func TestIssueDeriverReproducesIssueStageFor(t *testing.T) { {"hive-filed untriaged (015)", IssueStageInput{AuthorKind: authorkind.Hive}}, {"hive-filed accepted (010)", IssueStageInput{AuthorKind: authorkind.Hive, Labels: []string{"triage/accepted"}}}, {"human-filed needs-triage (015)", IssueStageInput{AuthorKind: authorkind.Human, Labels: []string{"needs-triage"}}}, + // hivecommons/hive#18141: the discussion signal travels through Facts + // (labels and title) so the adapter derives 018 exactly as the shim. + {"architecture discussion unclaimed (018)", IssueStageInput{Labels: []string{"Epic", "architecture discussion"}, Title: "Hold: v6 agent runtime"}}, + {"[RFC] title unclaimed (018)", IssueStageInput{Labels: []string{"Epic"}, Title: "📝 [RFC] v6 agent runtime"}}, + {"[RFC] title claimed (020)", IssueStageInput{Title: "[RFC] v6 agent runtime", Bucket: workBucketClaimed, ClaimKind: "human", ClaimedBy: "alice"}}, } deriver := IssueDeriver() if deriver.Kind() != stagelabels.IssueItem { diff --git a/src/pkg/github/issue_stage_goldens_test.go b/src/pkg/github/issue_stage_goldens_test.go index eb4007a3c9..6a3f44bd3d 100644 --- a/src/pkg/github/issue_stage_goldens_test.go +++ b/src/pkg/github/issue_stage_goldens_test.go @@ -43,13 +43,16 @@ const issueStageGoldenDir = "testdata/issue_stage_goldens" // PR's stage as the shared store would answer it ("" = no resolver, "none" // = resolver with no record). type issueStageGoldenInput struct { - Repo string `json:"repo"` - Number int `json:"number"` - Labels []string `json:"labels,omitempty"` - Bucket string `json:"bucket,omitempty"` - Reason string `json:"reason,omitempty"` - ClaimKind string `json:"claim_kind,omitempty"` - ClaimedBy string `json:"claimed_by,omitempty"` + Repo string `json:"repo"` + Number int `json:"number"` + Labels []string `json:"labels,omitempty"` + // Title mirrors IssueStageInput.Title: the RFC-title discussion signal + // (hivecommons/hive#18141) is the only derivation that reads it. + Title string `json:"title,omitempty"` + Bucket string `json:"bucket,omitempty"` + Reason string `json:"reason,omitempty"` + ClaimKind string `json:"claim_kind,omitempty"` + ClaimedBy string `json:"claimed_by,omitempty"` // PRStarted mirrors IssueStageInput.PRStarted: the live claim carries // PR evidence (hivecommons/hive#13069). PRStarted bool `json:"pr_started,omitempty"` @@ -82,7 +85,7 @@ type issueStageGoldenCase struct { } func (in issueStageGoldenInput) toInput() IssueStageInput { - out := IssueStageInput{Repo: in.Repo, Number: in.Number, Labels: in.Labels, Bucket: in.Bucket, Reason: in.Reason, ClaimKind: in.ClaimKind, ClaimedBy: in.ClaimedBy, PRStarted: in.PRStarted, AuthorKind: authorkind.Kind(in.AuthorKind)} + out := IssueStageInput{Repo: in.Repo, Number: in.Number, Labels: in.Labels, Title: in.Title, Bucket: in.Bucket, Reason: in.Reason, ClaimKind: in.ClaimKind, ClaimedBy: in.ClaimedBy, PRStarted: in.PRStarted, AuthorKind: authorkind.Kind(in.AuthorKind)} if in.ClaimPresent { out.Claim = &IssueClaimContext{PRNumber: in.ClaimPR, PRRepo: in.ClaimPRRepo} } @@ -322,8 +325,23 @@ func generateIssueStageGoldens() map[string][]issueStageGoldenCase { }, { Name: "regression/17054-rfc-epic-without-proposal-stays-010", - Note: "The rule keys on labels only: hivecommons/hive#17054 as labelled on 2026-11-08 (Epic, no proposal / kind/design) is still 010; the stage engine never reads titles.", - Input: issueStageGoldenInput{Repo: repo, Number: 17054, Labels: []string{"help wanted", "Epic", HumanAckLabel, stagelabels.Actionable.Label()}}, + Note: "hivecommons/hive#17054 as labelled on 2026-11-08 (Epic, no proposal / kind/design) with no RFC title marker is still 010: the title rule (hivecommons/hive#18141) needs an [RFC] / RFC: opener.", + Input: issueStageGoldenInput{Repo: repo, Number: 17054, Labels: []string{"help wanted", "Epic", HumanAckLabel, stagelabels.Actionable.Label()}, Title: "v5 to v6 agent runtime"}, + }, + { + Name: "regression/18068-architecture-discussion-is-018", + Note: "hivecommons/hive#18068 (a v6 design RFC labelled Epic + architecture discussion, body 'Hold - design discussion, not a task') derived 010 and showed as idle agent work; architecture discussion is a discussion label (hivecommons/hive#18141).", + Input: issueStageGoldenInput{Repo: repo, Number: 18068, Labels: []string{"Epic", "architecture discussion"}, Title: "Hold: v6 agent runtime design"}, + }, + { + Name: "regression/18141-rfc-title-without-label-is-018", + Note: "An RFC filed before any discussion label lands: an [RFC] title opener (after the leading emoji) is a discussion signal, reason 'rfc title' (hivecommons/hive#18141).", + Input: issueStageGoldenInput{Repo: repo, Number: 18141, Labels: []string{"Epic", "help wanted"}, Title: "📝 [RFC] v6 agent runtime"}, + }, + { + Name: "regression/18141-rfc-title-human-claim-is-020", + Note: "The title rule is not a gate either: a person's /claim on an [RFC]-titled issue stages 020.", + Input: issueStageGoldenInput{Repo: repo, Number: 18141, Labels: []string{"help wanted", stagelabels.OpenForDiscussion.Label()}, Title: "[RFC] v6 agent runtime", Bucket: workBucketClaimed, ClaimKind: string(claims.KindHuman), ClaimedBy: "alice"}, }, { Name: "regression/17150-kind-design-is-018", diff --git a/src/pkg/github/issue_stage_labels.go b/src/pkg/github/issue_stage_labels.go index 8f8a162abc..4bf1859d88 100644 --- a/src/pkg/github/issue_stage_labels.go +++ b/src/pkg/github/issue_stage_labels.go @@ -64,6 +64,7 @@ import ( "sort" "strings" "time" + "unicode" gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/pkg/authorkind" @@ -165,9 +166,13 @@ var triageAwaitingLabel = config.DefaultReporterTrustAwaitingLabel // IssueStageInput is the one shape both actionable items and work-detail // rows reduce to before classification. type IssueStageInput struct { - Repo string - Number int - Labels []string + Repo string + Number int + Labels []string + // Title is the issue title; the only derivation that reads it is the + // RFC-title discussion signal ("[RFC]" / "RFC:" opener stages an + // unlabelled RFC 018, hivecommons/hive#18141). + Title string Bucket string // "" for an issue still in the actionable set Reason string // the bucket's reason; for "filtered", the label that parked it ClaimKind string @@ -297,19 +302,30 @@ func parkedOnPersonLabel(in IssueStageInput) string { } // discussionLabels are the labels that mark an issue whose remaining -// content is a discussion — a proposal or a design question — rather than -// agent work. An open issue wearing one with no live claim derives -// stage/018-open-for-discussion instead of 010/015 -// (hivecommons/hive#17150). They are NOT claim-gate labels: admission -// lets anyone claim such an issue, and the claim stages 020/030. -var discussionLabels = []string{"proposal", "kind/design"} +// content is a discussion — a proposal, a design question or an +// architecture RFC — rather than agent work. An open issue wearing one +// with no live claim derives stage/018-open-for-discussion instead of +// 010/015 (hivecommons/hive#17150, hivecommons/hive#18141). Most specific +// first: the reason names the first match. They are NOT claim-gate labels: +// admission lets anyone claim such an issue, and the claim stages 020/030. +var discussionLabels = []string{"proposal", "kind/design", "architecture discussion"} + +// rfcTitleDiscussionReason is the IssueFacts.DiscussionLabel value for an +// issue whose title, not its labels, marks it as an RFC +// (hivecommons/hive#18141): the 018 reason reads +// "open for discussion: rfc title". +const rfcTitleDiscussionReason = "rfc title" + +// rfcTitlePrefixes are the title openers (after any leading emoji, +// case-insensitive) that mark an issue as an RFC before any label lands. +var rfcTitlePrefixes = []string{"[rfc]", "rfc:"} // issueDiscussionLabel returns the first discussionLabels entry labels // carry (case-insensitive, as the issue spells it), or "". func issueDiscussionLabel(labels []string) string { - for _, l := range labels { - name := strings.TrimSpace(l) - for _, want := range discussionLabels { + for _, want := range discussionLabels { + for _, l := range labels { + name := strings.TrimSpace(l) if strings.EqualFold(name, want) { return name } @@ -318,6 +334,35 @@ func issueDiscussionLabel(labels []string) string { return "" } +// issueDiscussionSignal is what IssueFacts.DiscussionLabel carries: the +// discussion label the issue wears, or rfcTitleDiscussionReason when it +// wears none but its title opens with an RFC marker, or "". +func issueDiscussionSignal(labels []string, title string) string { + if label := issueDiscussionLabel(labels); label != "" { + return label + } + if isRFCTitle(title) { + return rfcTitleDiscussionReason + } + return "" +} + +// isRFCTitle reports whether title opens with "[RFC]" or "RFC:" +// (case-insensitive) once any leading emoji, punctuation or whitespace is +// skipped, so "📝 [RFC] v6 ..." and "RFC: ..." both count. +func isRFCTitle(title string) bool { + rest := strings.TrimLeftFunc(title, func(r rune) bool { + return r != '[' && !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) + rest = strings.ToLower(rest) + for _, p := range rfcTitlePrefixes { + if strings.HasPrefix(rest, p) { + return true + } + } + return false +} + // issueAuthorKind is the stage input's author kind from the enumeration's // verdict (Client.isHumanAuthor): Human when it said so, Hive — the App // bot, a `[bot]` login or a hive identity — when it did not. The verdict @@ -355,7 +400,7 @@ func IssueStageInputs(result *ActionableResult) []IssueStageInput { } var out []IssueStageInput for _, issue := range result.Issues.Items { - out = append(out, IssueStageInput{Repo: issue.Repo, Number: issue.Number, Labels: forgeIssueLabels(issue.Labels, issue.ClaimContext), ClaimKind: issue.ClaimKind, ClaimedBy: issue.ClaimedBy, PRStarted: issue.ClaimPRStarted, Claim: issue.ClaimContext, Children: issue.Children, AuthorKind: issueAuthorKind(issue), DelegatesWork: hasUnfinishedDelegatedWork(issue.Body), SpekUmbrella: spekmarker.IsUmbrella(issue.Body), UpdatedAt: issue.UpdatedAt, Assignees: issue.Assignees}) + out = append(out, IssueStageInput{Repo: issue.Repo, Number: issue.Number, Labels: forgeIssueLabels(issue.Labels, issue.ClaimContext), Title: issue.Title, ClaimKind: issue.ClaimKind, ClaimedBy: issue.ClaimedBy, PRStarted: issue.ClaimPRStarted, Claim: issue.ClaimContext, Children: issue.Children, AuthorKind: issueAuthorKind(issue), DelegatesWork: hasUnfinishedDelegatedWork(issue.Body), SpekUmbrella: spekmarker.IsUmbrella(issue.Body), UpdatedAt: issue.UpdatedAt, Assignees: issue.Assignees}) } // Held issues leave the pipeline at enumeration (Hold list, no // work-detail row). They still have to be walked so a stage label @@ -378,7 +423,7 @@ func IssueStageInputs(result *ActionableResult) []IssueStageInput { if row.Number <= 0 { continue } - out = append(out, IssueStageInput{Repo: row.Repo, Number: row.Number, Labels: forgeIssueLabels(row.Labels, row.ClaimContext), Bucket: row.Bucket, Reason: row.Reason, ClaimKind: row.ClaimKind, ClaimedBy: row.ClaimedBy, PRStarted: row.ClaimPRStarted, Claim: row.ClaimContext, Children: row.Children, UpdatedAt: row.UpdatedAt, Assignees: row.Assignees}) + out = append(out, IssueStageInput{Repo: row.Repo, Number: row.Number, Labels: forgeIssueLabels(row.Labels, row.ClaimContext), Title: row.Title, Bucket: row.Bucket, Reason: row.Reason, ClaimKind: row.ClaimKind, ClaimedBy: row.ClaimedBy, PRStarted: row.ClaimPRStarted, Claim: row.ClaimContext, Children: row.Children, UpdatedAt: row.UpdatedAt, Assignees: row.Assignees}) } } return out diff --git a/src/pkg/github/issue_stage_labels_test.go b/src/pkg/github/issue_stage_labels_test.go index f5d8110a27..2e069389c4 100644 --- a/src/pkg/github/issue_stage_labels_test.go +++ b/src/pkg/github/issue_stage_labels_test.go @@ -312,6 +312,71 @@ func issueStageTestClient(t *testing.T, srv *httptest.Server, enabled bool) *Cli func stageNow() time.Time { return time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) } +// TestIssueDiscussionSignal pins the 018 discussion signal +// (hivecommons/hive#17150, hivecommons/hive#18141): the most specific +// discussion label wins and is reported as the issue spells it; an +// unlabelled issue whose title opens with "[RFC]" / "RFC:" (after any +// leading emoji) reads "rfc title"; anything else is "". +func TestIssueDiscussionSignal(t *testing.T) { + cases := []struct { + name string + labels []string + title string + want string + }{ + {"no signal", []string{"Epic", "help wanted"}, "v6 design: agent runtime", ""}, + {"proposal", []string{"proposal"}, "", "proposal"}, + {"kind/design as spelled", []string{"Kind/Design"}, "", "Kind/Design"}, + {"architecture discussion", []string{"Epic", "architecture discussion"}, "Hold: v6 runtime", "architecture discussion"}, + {"architecture discussion case-insensitive", []string{"Architecture Discussion"}, "", "Architecture Discussion"}, + {"most specific label wins over order", []string{"architecture discussion", "proposal"}, "", "proposal"}, + {"label outranks rfc title", []string{"kind/design"}, "[RFC] v6 agent runtime", "kind/design"}, + {"[RFC] title", nil, "[RFC] v6 agent runtime", rfcTitleDiscussionReason}, + {"RFC: title", nil, "RFC: v6 agent runtime", rfcTitleDiscussionReason}, + {"lower-case rfc title", nil, "[rfc] v6 agent runtime", rfcTitleDiscussionReason}, + {"emoji-prefixed RFC title", []string{"Epic"}, "📝 [RFC] v6 agent runtime", rfcTitleDiscussionReason}, + {"emoji and spaces before RFC:", nil, " 🧭 RFC: v6 agent runtime", rfcTitleDiscussionReason}, + {"rfc word without marker", nil, "RFC process cleanup", ""}, + {"rfc mid-title", nil, "Implement the [RFC] runtime", ""}, + {"empty title", nil, "", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := issueDiscussionSignal(tc.labels, tc.title); got != tc.want { + t.Fatalf("issueDiscussionSignal(%q, %q) = %q; want %q", tc.labels, tc.title, got, tc.want) + } + }) + } +} + +// TestIssueStageFor_DiscussionSignals pins the 018 derivation end to end +// for the hivecommons/hive#18141 shapes: an unclaimed issue with only +// `architecture discussion`, an unclaimed `[RFC]`-titled issue with no +// discussion label, and the same issues once claimed (020 / 030). +func TestIssueStageFor_DiscussionSignals(t *testing.T) { + cases := []struct { + name string + in IssueStageInput + want stagelabels.Stage + wantReason string + }{ + {"architecture discussion unclaimed", IssueStageInput{Labels: []string{"Epic", "architecture discussion"}, Title: "Hold: v6 agent runtime"}, stagelabels.OpenForDiscussion, "open for discussion: architecture discussion"}, + {"[RFC] title, no discussion label", IssueStageInput{Labels: []string{"Epic", "help wanted"}, Title: "📝 [RFC] v6 agent runtime"}, stagelabels.OpenForDiscussion, "open for discussion: rfc title"}, + {"RFC: title, no labels", IssueStageInput{Title: "RFC: v6 agent runtime"}, stagelabels.OpenForDiscussion, "open for discussion: rfc title"}, + {"[RFC] title claimed by a person", IssueStageInput{Title: "[RFC] v6 agent runtime", Bucket: workBucketClaimed, ClaimKind: "human", ClaimedBy: "alice"}, stagelabels.Claimed, "live issue claim"}, + {"architecture discussion claimed with a PR under way", IssueStageInput{Labels: []string{"architecture discussion"}, Title: "Hold: v6 agent runtime", Bucket: workBucketClaimed, ClaimKind: string(claims.KindAgent), ClaimedBy: "scanner", PRStarted: true}, stagelabels.PRInProgress, "live claim with a PR under way"}, + {"plain title stays 010", IssueStageInput{Labels: []string{"Epic", "help wanted"}, Title: "v6 agent runtime"}, stagelabels.Actionable, "actionable"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stage, covered, reason := IssueStageFor(tc.in, nil) + if stage != tc.want || covered || reason != tc.wantReason { + t.Fatalf("got stage=%v covered=%v reason=%q; want stage=%v reason=%q", stage, covered, reason, tc.want, tc.wantReason) + } + }) + } +} + func TestIssueStageFor(t *testing.T) { prStage := func(repo string, number int) stagelabels.Stage { if repo == "acme/widgets" && number == 99 { diff --git a/src/pkg/github/testdata/issue_stage_goldens/regressions.json b/src/pkg/github/testdata/issue_stage_goldens/regressions.json index dd419ae84d..b3bfd560c7 100644 --- a/src/pkg/github/testdata/issue_stage_goldens/regressions.json +++ b/src/pkg/github/testdata/issue_stage_goldens/regressions.json @@ -11,11 +11,14 @@ {"name":"regression/13069-bare-agent-claim-derives-020","note":"hivecommons/hive#13069 wore stage/030-pr-in-progress on the scanner's `hive-open-issue claim` with no PR, branch or PR request anywhere; an agent claim without PR evidence is 020-claimed.","input":{"repo":"hivecommons/hive","number":13069,"labels":["kind/bug","stage/030-pr-in-progress","hive/claimed-by-scanner"],"bucket":"claimed","claim_kind":"agent","claimed_by":"scanner"},"want":{"stage":"020-claimed","covered":false,"reason":"live agent claim (no PR yet)"}}, {"name":"regression/17020-proposal-after-refs-pr-is-018","note":"hivecommons/hive#17020 (label proposal): its Refs PR merged and close-on-merge left hive/partially-addressed; the remainder is a discussion, so it derives 018-open-for-discussion, not 010 (hivecommons/hive#17150).","input":{"repo":"hivecommons/hive","number":17020,"labels":["good first issue","help wanted","hive/verified-open","proposal","hive/partially-addressed"]},"want":{"stage":"018-open-for-discussion","covered":false,"reason":"open for discussion: proposal"}}, {"name":"regression/17054-rfc-epic-labelled-proposal-is-018","note":"hivecommons/hive#17054 (a 📝 RFC whose Refs PR merged) derives 018 once it carries proposal.","input":{"repo":"hivecommons/hive","number":17054,"labels":["help wanted","Epic","approved-direction","proposal"]},"want":{"stage":"018-open-for-discussion","covered":false,"reason":"open for discussion: proposal"}}, -{"name":"regression/17054-rfc-epic-without-proposal-stays-010","note":"The rule keys on labels only: hivecommons/hive#17054 as labelled on 2026-11-08 (Epic, no proposal / kind/design) is still 010; the stage engine never reads titles.","input":{"repo":"hivecommons/hive","number":17054,"labels":["help wanted","Epic","approved-direction","stage/010-actionable"]},"want":{"stage":"010-actionable","covered":false,"reason":"actionable"}}, +{"name":"regression/17054-rfc-epic-without-proposal-stays-010","note":"hivecommons/hive#17054 as labelled on 2026-11-08 (Epic, no proposal / kind/design) with no RFC title marker is still 010: the title rule (hivecommons/hive#18141) needs an [RFC] / RFC: opener.","input":{"repo":"hivecommons/hive","number":17054,"labels":["help wanted","Epic","approved-direction","stage/010-actionable"],"title":"v5 to v6 agent runtime"},"want":{"stage":"010-actionable","covered":false,"reason":"actionable"}}, {"name":"regression/17150-kind-design-is-018","input":{"repo":"hivecommons/hive","number":17150,"labels":["Kind/Design"]},"want":{"stage":"018-open-for-discussion","covered":false,"reason":"open for discussion: Kind/Design"}}, {"name":"regression/17150-proposal-agent-claim-with-pr-is-030","input":{"repo":"hivecommons/hive","number":17150,"labels":["proposal"],"bucket":"claimed","claim_kind":"agent","claimed_by":"scanner","pr_started":true},"want":{"stage":"030-pr-in-progress","covered":false,"reason":"live claim with a PR under way"}}, {"name":"regression/17150-proposal-human-claim-is-020","note":"018 is not a gate: a person's /claim stages 020 as on any issue.","input":{"repo":"hivecommons/hive","number":17150,"labels":["proposal","stage/018-open-for-discussion"],"bucket":"claimed","claim_kind":"human","claimed_by":"alice"},"want":{"stage":"020-claimed","covered":false,"reason":"live issue claim"}}, {"name":"regression/17150-proposal-label-removed-is-010","note":"Removing proposal / kind/design returns the issue to plain 010 derivation, whatever stage label it still wears.","input":{"repo":"hivecommons/hive","number":17150,"labels":["help wanted","stage/018-open-for-discussion"]},"want":{"stage":"010-actionable","covered":false,"reason":"actionable"}}, +{"name":"regression/18068-architecture-discussion-is-018","note":"hivecommons/hive#18068 (a v6 design RFC labelled Epic + architecture discussion, body 'Hold - design discussion, not a task') derived 010 and showed as idle agent work; architecture discussion is a discussion label (hivecommons/hive#18141).","input":{"repo":"hivecommons/hive","number":18068,"labels":["Epic","architecture discussion"],"title":"Hold: v6 agent runtime design"},"want":{"stage":"018-open-for-discussion","covered":false,"reason":"open for discussion: architecture discussion"}}, +{"name":"regression/18141-rfc-title-human-claim-is-020","note":"The title rule is not a gate either: a person's /claim on an [RFC]-titled issue stages 020.","input":{"repo":"hivecommons/hive","number":18141,"labels":["help wanted","stage/018-open-for-discussion"],"title":"[RFC] v6 agent runtime","bucket":"claimed","claim_kind":"human","claimed_by":"alice"},"want":{"stage":"020-claimed","covered":false,"reason":"live issue claim"}}, +{"name":"regression/18141-rfc-title-without-label-is-018","note":"An RFC filed before any discussion label lands: an [RFC] title opener (after the leading emoji) is a discussion signal, reason 'rfc title' (hivecommons/hive#18141).","input":{"repo":"hivecommons/hive","number":18141,"labels":["Epic","help wanted"],"title":"📝 [RFC] v6 agent runtime"},"want":{"stage":"018-open-for-discussion","covered":false,"reason":"open for discussion: rfc title"}}, {"name":"regression/agent-claim-kind-is-case-insensitive","input":{"repo":"hivecommons/hive","number":12456,"bucket":"claimed","claim_kind":"Agent"},"want":{"stage":"020-claimed","covered":false,"reason":"live agent claim (no PR yet)"}}, {"name":"regression/covered-pr-before-040-is-not-covered","note":"A covering PR the store still has at an issue stage (\u003c040) does not count as staged coverage.","input":{"repo":"hivecommons/hive","number":12452,"bucket":"claimed_by_pr","claim_present":true,"claim_pr":12461,"covering_pr_stage":"020-claimed"},"want":{"stage":"040-covered-by-pr","covered":false,"reason":"open PR covers it","lookup":"hivecommons/hive#12461"}}, {"name":"regression/covered-pr-in-other-repo-is-looked-up-there","note":"IssueClaimContext.PRRepo overrides the issue's repo for the store lookup.","input":{"repo":"hivecommons/hive","number":12453,"bucket":"claimed_by_pr","claim_present":true,"claim_pr":5,"claim_pr_repo":"hivecommons/docs","covering_pr_stage":"100-merge-queue"},"want":{"stage":"040-covered-by-pr","covered":true,"reason":"open PR #5 covers it (PR at 100-merge-queue)","lookup":"hivecommons/docs#5"}}, diff --git a/src/pkg/stageengine/derive_issue.go b/src/pkg/stageengine/derive_issue.go index fa88ffb31d..3594ef365d 100644 --- a/src/pkg/stageengine/derive_issue.go +++ b/src/pkg/stageengine/derive_issue.go @@ -43,8 +43,9 @@ func ReasonIsPartialCover(reason string) bool { return strings.HasPrefix(strings.TrimSpace(reason), "open PR addresses part of it") } -// ReasonDiscussionLabel reads the proposal / design label an 018 -// open-for-discussion reason names ("open for discussion: proposal"), or "" +// ReasonDiscussionLabel reads the discussion signal an 018 +// open-for-discussion reason names ("open for discussion: proposal", or +// "open for discussion: rfc title" for a title-only RFC), or "" // when the reason is not one, so presentation layers can name the label // without re-deriving the stage. func ReasonDiscussionLabel(reason string) string { diff --git a/src/pkg/stageengine/derive_issue_test.go b/src/pkg/stageengine/derive_issue_test.go index 25143426a8..49feede54c 100644 --- a/src/pkg/stageengine/derive_issue_test.go +++ b/src/pkg/stageengine/derive_issue_test.go @@ -102,6 +102,24 @@ func TestDeriveIssueTable(t *testing.T) { // discussion, not idle agent work; a live claim stages it as usual. {"proposal unclaimed", func(f *Facts) { f.Issue.DiscussionLabel = "proposal" }, stagelabels.OpenForDiscussion, false, "open for discussion: proposal"}, {"kind/design unclaimed", func(f *Facts) { f.Issue.DiscussionLabel = "kind/design" }, stagelabels.OpenForDiscussion, false, "open for discussion: kind/design"}, + // hivecommons/hive#18141: `architecture discussion` and a title-only + // RFC signal ("rfc title") derive 018 the same way; a claim still wins. + {"architecture discussion unclaimed", func(f *Facts) { f.Issue.DiscussionLabel = "architecture discussion" }, stagelabels.OpenForDiscussion, false, "open for discussion: architecture discussion"}, + {"rfc title unclaimed", func(f *Facts) { f.Issue.DiscussionLabel = "rfc title" }, stagelabels.OpenForDiscussion, false, "open for discussion: rfc title"}, + {"rfc title untriaged", func(f *Facts) { + f.Issue.DiscussionLabel = "rfc title" + f.Issue.TriageAwaiting = "carries needs-triage" + }, stagelabels.OpenForDiscussion, false, "open for discussion: rfc title"}, + {"rfc title claimed", func(f *Facts) { + f.Issue.DiscussionLabel = "rfc title" + f.Issue.Bucket = BucketClaimed + f.Issue.Claim = &ClaimFact{Holder: "alice", Kind: "human"} + }, stagelabels.Claimed, false, "live issue claim"}, + {"architecture discussion claimed with a PR under way", func(f *Facts) { + f.Issue.DiscussionLabel = "architecture discussion" + f.Issue.Bucket = BucketClaimed + f.Issue.Claim = &ClaimFact{Kind: claims.KindAgent, PRStarted: true} + }, stagelabels.PRInProgress, false, "live claim with a PR under way"}, {"proposal outranks triage", func(f *Facts) { f.Issue.DiscussionLabel = "proposal" f.Issue.TriageAwaiting = "carries needs-triage" diff --git a/src/pkg/stageengine/facts.go b/src/pkg/stageengine/facts.go index a784b5ccfd..13a6678332 100644 --- a/src/pkg/stageengine/facts.go +++ b/src/pkg/stageengine/facts.go @@ -54,15 +54,19 @@ type Facts struct { Ref Ref ObservedAt time.Time Labels []string // list payload or fresh read; includes non-hive labels - Open bool - Merged bool - Draft bool - Author AuthorFacts - Park ParkFacts - Adoption AdoptionFacts - Origin string // the origin/<source> label, "" for a human filing - Issue *IssueFacts - PR *PRFacts + // Title is the item's title when the producer had it, "" otherwise. + // Derivers do not read it; pkg/github resolves an issue's RFC-title + // discussion signal from it (hivecommons/hive#18141). + Title string + Open bool + Merged bool + Draft bool + Author AuthorFacts + Park ParkFacts + Adoption AdoptionFacts + Origin string // the origin/<source> label, "" for a human filing + Issue *IssueFacts + PR *PRFacts } // AuthorFacts is who wrote the item. @@ -180,11 +184,13 @@ type IssueFacts struct { // UmbrellaLabel is the never-claim label the issue carries // (governor.claims.never_claim_labels, default `epic`). UmbrellaLabel string - // DiscussionLabel is the proposal / design label the issue carries - // (`proposal`, `kind/design`), "" when it carries none. An unclaimed - // issue wearing one derives 018-open-for-discussion instead of - // 010/015; a live claim or covering PR stages it as on any issue - // (hivecommons/hive#17150). + // DiscussionLabel is the discussion signal the issue carries: the + // proposal / design label (`proposal`, `kind/design`, `architecture + // discussion`) or `rfc title` for an unlabelled issue whose title opens + // with "[RFC]" / "RFC:"; "" when it carries none. An unclaimed issue + // with one derives 018-open-for-discussion instead of 010/015; a live + // claim or covering PR stages it as on any issue + // (hivecommons/hive#17150, hivecommons/hive#18141). DiscussionLabel string // TriageAwaiting is why an actionable issue is still untriaged (015): // the reporter-trust awaiting label it carries or the gate's From 6df6262e3d8c74951e499e1d75df164c80df808d Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 19:06:45 -0500 Subject: [PATCH 189/223] =?UTF-8?q?=E2=9C=A8=20hub:=20self-service=20(byo-?= =?UTF-8?q?)=20hive=20registry=20with=20hashed=20rotatable=20keys=20and=20?= =?UTF-8?q?bearer=20verification=20(#18150)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub: self-service (byo-) hive registry with hashed rotatable keys and bearer verification Step 2 of the #18060 RFC. A self-service hive's heartbeat bearer is a random key the hub mints per hive and stores only as a SHA-256 hash in /data/hub-selfservice.json. verifyHeartbeatBearerGeneration checks a byo- ID against that hash and nothing else: the master-derived paths are never tried for a byo- ID, and a self-service key is only ever compared against the record for the ID the caller claims (audit F2). Rotation keeps the previous hash live for a 10-minute overlap; revoke removes the record. Accepted heartbeats stamp the record for the expiry sweep that follows in a later child. Fixes #18142 Part of #18060 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 📖 docs: resync api-reference citations after hub server.go line shift Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 hub: debounce self-service heartbeat stamp against the last persisted value Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- .../added-18060-selfservice-registry-keys.md | 1 + src/docs/api-reference/hub-server.md | 58 +-- src/pkg/hub/hub_keys.go | 15 + src/pkg/hub/selfservice_registry.go | 371 ++++++++++++++++++ src/pkg/hub/selfservice_registry_test.go | 341 ++++++++++++++++ src/pkg/hub/server.go | 24 +- 6 files changed, 779 insertions(+), 31 deletions(-) create mode 100644 changelog.d/added-18060-selfservice-registry-keys.md create mode 100644 src/pkg/hub/selfservice_registry.go create mode 100644 src/pkg/hub/selfservice_registry_test.go diff --git a/changelog.d/added-18060-selfservice-registry-keys.md b/changelog.d/added-18060-selfservice-registry-keys.md new file mode 100644 index 0000000000..82dd199f16 --- /dev/null +++ b/changelog.d/added-18060-selfservice-registry-keys.md @@ -0,0 +1 @@ +- Hub: self-service (`byo-`) hives now have their own heartbeat credential ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18142](https://github.com/hivecommons/hive/issues/18142)). The hub mints a random key per registered hive, stores only its SHA-256 hash in `/data/hub-selfservice.json`, and verifies a `byo-` heartbeat against that hash and nothing else: a master-derived key never authenticates a `byo-` ID, and a self-service key authenticates no other hive. Keys rotate with a 10-minute overlap so a redeploy does not show the hive offline, and each accepted heartbeat stamps the record for the coming expiry sweep. The registration API and UI follow in later PRs; nothing changes for hosted or operator-provisioned hives. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 9a605a3930..c13fed2c42 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,42 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks/<hash>/<YYYY-MM>.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1795` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1796` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1807` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1808` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1727` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1728` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1729` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1730` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1731` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1742` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1743` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1744` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1745` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1739` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1740` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1741` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1742` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1743` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1754` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1755` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1756` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1757` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=<id>[,<id>…]` (repeatable) and `?level=<n>` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence-<hive|fleet>-<from>-<to>.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#<n>`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1753` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1763` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1764` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1765` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1766` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1767` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1768` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1776` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1777` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1784` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1786` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1803` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1808` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1810` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1765` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1775` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1776` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1777` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1778` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1779` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1780` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1788` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1789` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1796` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1798` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1815` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1820` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1822` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/pkg/hub/hub_keys.go b/src/pkg/hub/hub_keys.go index 4a0d8dce19..3221487fa4 100644 --- a/src/pkg/hub/hub_keys.go +++ b/src/pkg/hub/hub_keys.go @@ -199,6 +199,17 @@ func (s *HubServer) verifyHeartbeatBearerGeneration(presented, hiveID string) (i if s == nil || presented == "" { return 0, false } + // Self-service (byo-) hives (#18060): the ONLY accepted credential is the + // random key the hub minted for exactly this ID, checked against its stored + // hash. The master-derived paths below are never tried for a byo- ID, so no + // holder of HIVE_HUB_SECRET or a master-derived key can beat as one, and a + // self-service key is never compared against any other hive's record. + if IsSelfServiceHiveID(hiveID) { + if s.selfService.verify(hiveID, presented) { + return selfServiceBearerGeneration, true + } + return 0, false + } if s.keyGenerations != nil { return verifyHeartbeatBearerAcrossGenerations(s.keyGenerations, presented, hiveID, time.Now()) } @@ -300,6 +311,10 @@ func (s *HubServer) heartbeatBearerIsPerHive(presented, hiveID string) bool { if s == nil { return false } + // A self-service key is bound to one hive ID by construction (#18060). + if IsSelfServiceHiveID(hiveID) { + return s.selfService.verify(hiveID, presented) + } if s.keyGenerations != nil { _, ok := verifyHeartbeatBearerAcrossGenerations(s.keyGenerations, presented, hiveID, time.Now()) return ok diff --git a/src/pkg/hub/selfservice_registry.go b/src/pkg/hub/selfservice_registry.go new file mode 100644 index 0000000000..44bb2e62a2 --- /dev/null +++ b/src/pkg/hub/selfservice_registry.go @@ -0,0 +1,371 @@ +package hub + +// Self-service (byo-) hive registry: random, hashed, rotatable heartbeat keys +// (#18060, step 2 of the RFC; step 1 is heartbeat_selfservice_fence.go). +// +// A hive in the reserved SelfServiceHiveIDPrefix namespace runs on its owner's +// infrastructure. Its heartbeat bearer is NOT derived from the hub master +// (derivePerHiveKey): the owner must never hold HIVE_HUB_SECRET, a +// master-derived key cannot be rotated without a new hive ID (#1537), and a +// master rotation would break hives the operator cannot redeploy. Instead the +// hub mints a random key per hive, stores only its SHA-256 hash, and shows the +// raw key to the owner exactly once (the registration API, a later child). +// +// Rotation keeps the previous hash live for selfServiceKeyRotationOverlap so a +// redeploy with the new key does not show the hive offline — the same idea as +// maxLiveGenerations for the master. +// +// Audit F2: each bearer identifies exactly one hive. verify() looks a key up BY +// hive ID and compares against that record only, so a self-service key can +// never authenticate any other ID, and verifyHeartbeatBearerGeneration never +// tries the master-derived path for a byo- ID. +// +// Key material never leaves this file except as the one-time return value of +// register and rotate. Nothing here logs a raw key or a hash. + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// selfServiceStorePath is the DEFAULT on-disk self-service registry. A var so +// tests can redirect it, like registryPath. +var selfServiceStorePath = "/data/hub-selfservice.json" + +const ( + // selfServiceKeyBytes is the entropy of a minted heartbeat key. + selfServiceKeyBytes = 32 + // selfServiceKeyRotationOverlap is how long the previous key keeps + // authenticating after a rotation, so the owner can redeploy the spoke + // with the new key without the hive going offline. + selfServiceKeyRotationOverlap = 10 * time.Minute + // selfServiceHeartbeatStampInterval bounds how often an accepted heartbeat + // rewrites the store: the stamp exists for the expiry sweep (24h + // granularity), so persisting it on every beat would be pure churn. + selfServiceHeartbeatStampInterval = time.Minute + // selfServiceBearerGeneration is the generation ID reported by + // verifyHeartbeatBearerGeneration for a key verified against this store. + // It is not a master generation; it only has to differ from the master + // generation IDs so the auth-rollout view can tell the lanes apart. + selfServiceBearerGeneration = -1 + selfServiceStoreFileMode = 0o600 + selfServiceStoreDirMode = 0o755 +) + +var ( + errSelfServiceNotByoID = errors.New("self-service hive ID must start with " + SelfServiceHiveIDPrefix) + errSelfServiceExists = errors.New("self-service hive already registered") + errSelfServiceNotFound = errors.New("self-service hive not found") + errSelfServiceNilStore = errors.New("self-service store not configured") + errSelfServiceEmptyOwner = errors.New("self-service hive owner required") +) + +// SelfServiceHive is one owner-registered hive. Only hashes of its keys are +// ever stored; the raw key exists in memory for the duration of the register +// or rotate call that minted it. +type SelfServiceHive struct { + ID string `json:"id"` + Owner string `json:"owner"` // canonical hub login of the registering user + Org string `json:"org,omitempty"` + // Repos the owner proved admin permission on at OwnershipCheckedAt. + Repos []string `json:"repos,omitempty"` + DashboardURL string `json:"dashboard_url,omitempty"` + // KeyHash is the SHA-256 (hex) of the current heartbeat key. + KeyHash string `json:"key_hash"` + // PrevKeyHash is the hash of the key before the last rotation; it keeps + // authenticating until PrevKeyExpiresAt. + PrevKeyHash string `json:"prev_key_hash,omitempty"` + PrevKeyExpiresAt time.Time `json:"prev_key_expires_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + // OwnershipCheckedAt is when the hub last confirmed the owner's admin + // permission on Repos. Zero when never checked. + OwnershipCheckedAt time.Time `json:"ownership_checked_at,omitempty"` + // LastAcceptedHeartbeatAt is the last heartbeat that passed the bearer + // check. Zero until the first one; the expiry sweep (a later child) + // removes registrations that never reach it. + LastAcceptedHeartbeatAt time.Time `json:"last_accepted_heartbeat_at,omitempty"` +} + +// selfServiceStore holds every SelfServiceHive and persists them as one JSON +// file via temp-file + rename, like saveRegistryNow. All methods are safe on a +// nil receiver (bare &HubServer{} test literals leave the field nil): reads +// answer "nothing registered" and writes return errSelfServiceNilStore. +type selfServiceStore struct { + mu sync.RWMutex + path string + hives map[string]*SelfServiceHive + // stampPersisted is, per hive, the LastAcceptedHeartbeatAt value last + // written to disk; noteAcceptedHeartbeat debounces against it, not against + // the in-memory stamp, so a steady stream of beats still flushes once per + // selfServiceHeartbeatStampInterval. + stampPersisted map[string]time.Time + // now is injectable for rotation-overlap tests; nil means time.Now. + now func() time.Time +} + +func newSelfServiceStore(path string) *selfServiceStore { + return &selfServiceStore{path: path, hives: make(map[string]*SelfServiceHive), stampPersisted: make(map[string]time.Time)} +} + +func (st *selfServiceStore) clock() time.Time { + if st != nil && st.now != nil { + return st.now() + } + return time.Now() +} + +type selfServiceStoreFile struct { + Hives []SelfServiceHive `json:"hives"` + UpdatedAt time.Time `json:"updated_at"` +} + +// load reads the store from disk. A missing file is not an error: the hub +// starts with no self-service hives. +func (st *selfServiceStore) load() error { + if st == nil { + return errSelfServiceNilStore + } + data, err := os.ReadFile(st.path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("read self-service registry: %w", err) + } + var file selfServiceStoreFile + if err := json.Unmarshal(data, &file); err != nil { + return fmt.Errorf("parse self-service registry: %w", err) + } + st.mu.Lock() + defer st.mu.Unlock() + st.hives = make(map[string]*SelfServiceHive, len(file.Hives)) + st.stampPersisted = make(map[string]time.Time, len(file.Hives)) + for i := range file.Hives { + h := file.Hives[i] + if h.ID == "" { + continue + } + st.hives[h.ID] = &h + if !h.LastAcceptedHeartbeatAt.IsZero() { + st.stampPersisted[h.ID] = h.LastAcceptedHeartbeatAt + } + } + return nil +} + +// saveLocked writes the store atomically. Caller holds st.mu (read or write). +func (st *selfServiceStore) saveLocked() error { + file := selfServiceStoreFile{Hives: make([]SelfServiceHive, 0, len(st.hives)), UpdatedAt: st.clock().UTC()} + for _, h := range st.hives { + file.Hives = append(file.Hives, *h) + } + sort.Slice(file.Hives, func(i, j int) bool { return file.Hives[i].ID < file.Hives[j].ID }) + data, err := json.MarshalIndent(file, "", " ") + if err != nil { + return fmt.Errorf("marshal self-service registry: %w", err) + } + if err := os.MkdirAll(filepath.Dir(st.path), selfServiceStoreDirMode); err != nil { + return fmt.Errorf("create self-service registry dir: %w", err) + } + tmp := st.path + ".tmp" + if err := os.WriteFile(tmp, data, selfServiceStoreFileMode); err != nil { + return fmt.Errorf("write self-service registry: %w", err) + } + if err := os.Rename(tmp, st.path); err != nil { + return fmt.Errorf("rename self-service registry: %w", err) + } + return nil +} + +// mintSelfServiceKey returns a fresh random heartbeat key (base64url, no +// padding) and its hash. +func mintSelfServiceKey() (raw, hash string, err error) { + b := make([]byte, selfServiceKeyBytes) + if _, err := rand.Read(b); err != nil { + return "", "", fmt.Errorf("mint self-service key: %w", err) + } + raw = base64.RawURLEncoding.EncodeToString(b) + return raw, hashSelfServiceKey(raw), nil +} + +// hashSelfServiceKey is the only transform applied to a key before storage. +// SHA-256 is sufficient: the input is 256 bits of CSPRNG output, so there is +// nothing for a dictionary or a rainbow table to find. +func hashSelfServiceKey(raw string) string { + sum := sha256.Sum256([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +// register adds a new self-service hive and returns its raw key ONCE. The ID +// must already be in the byo- namespace; callers (the registration API) build +// it as byo-<login>-<name> and run the registry-collision and validNamePattern +// checks before calling. +func (st *selfServiceStore) register(h SelfServiceHive) (string, error) { + if st == nil { + return "", errSelfServiceNilStore + } + h.ID = strings.ToLower(strings.TrimSpace(h.ID)) + if !IsSelfServiceHiveID(h.ID) { + return "", errSelfServiceNotByoID + } + if strings.TrimSpace(h.Owner) == "" { + return "", errSelfServiceEmptyOwner + } + raw, hash, err := mintSelfServiceKey() + if err != nil { + return "", err + } + st.mu.Lock() + defer st.mu.Unlock() + if _, exists := st.hives[h.ID]; exists { + return "", errSelfServiceExists + } + h.KeyHash = hash + h.PrevKeyHash = "" + h.PrevKeyExpiresAt = time.Time{} + h.CreatedAt = st.clock().UTC() + h.LastAcceptedHeartbeatAt = time.Time{} + st.hives[h.ID] = &h + if err := st.saveLocked(); err != nil { + delete(st.hives, h.ID) + return "", err + } + return raw, nil +} + +// rotate mints a new key for id and returns it ONCE. The previous key keeps +// authenticating until selfServiceKeyRotationOverlap has passed. +func (st *selfServiceStore) rotate(id string) (string, error) { + if st == nil { + return "", errSelfServiceNilStore + } + raw, hash, err := mintSelfServiceKey() + if err != nil { + return "", err + } + st.mu.Lock() + defer st.mu.Unlock() + h, ok := st.hives[id] + if !ok { + return "", errSelfServiceNotFound + } + before := *h + h.PrevKeyHash = h.KeyHash + h.PrevKeyExpiresAt = st.clock().UTC().Add(selfServiceKeyRotationOverlap) + h.KeyHash = hash + if err := st.saveLocked(); err != nil { + *h = before + return "", err + } + return raw, nil +} + +// revoke removes id. After it, no key authenticates the hive. +func (st *selfServiceStore) revoke(id string) (bool, error) { + if st == nil { + return false, errSelfServiceNilStore + } + st.mu.Lock() + defer st.mu.Unlock() + h, ok := st.hives[id] + if !ok { + return false, nil + } + delete(st.hives, id) + if err := st.saveLocked(); err != nil { + st.hives[id] = h + return false, err + } + delete(st.stampPersisted, id) + return true, nil +} + +// verify reports whether presented is the current key for id, or the previous +// key inside its overlap window. Constant-time on both comparisons. A key is +// only ever compared against the record for the ID the caller claims, so it +// authenticates that one hive and nothing else (audit F2). +func (st *selfServiceStore) verify(id, presented string) bool { + if st == nil || id == "" || presented == "" { + return false + } + st.mu.RLock() + defer st.mu.RUnlock() + h, ok := st.hives[id] + if !ok { + return false + } + hash := hashSelfServiceKey(presented) + if h.KeyHash != "" && secureCompareHub(hash, h.KeyHash) { + return true + } + if h.PrevKeyHash != "" && st.clock().Before(h.PrevKeyExpiresAt) && secureCompareHub(hash, h.PrevKeyHash) { + return true + } + return false +} + +// get returns a copy of the record for id. +func (st *selfServiceStore) get(id string) (SelfServiceHive, bool) { + if st == nil { + return SelfServiceHive{}, false + } + st.mu.RLock() + defer st.mu.RUnlock() + h, ok := st.hives[id] + if !ok { + return SelfServiceHive{}, false + } + return *h, true +} + +// list returns copies of every record, sorted by ID. +func (st *selfServiceStore) list() []SelfServiceHive { + if st == nil { + return nil + } + st.mu.RLock() + defer st.mu.RUnlock() + out := make([]SelfServiceHive, 0, len(st.hives)) + for _, h := range st.hives { + out = append(out, *h) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// noteAcceptedHeartbeat stamps LastAcceptedHeartbeatAt for id. It persists at +// most once per selfServiceHeartbeatStampInterval per hive; the in-memory +// stamp is always current. Unknown IDs are ignored: a byo- ID that is not in +// the store cannot have passed the bearer check. +func (st *selfServiceStore) noteAcceptedHeartbeat(id string, now time.Time) error { + if st == nil { + return nil + } + st.mu.Lock() + defer st.mu.Unlock() + h, ok := st.hives[id] + if !ok { + return nil + } + h.LastAcceptedHeartbeatAt = now.UTC() + persisted, wasPersisted := st.stampPersisted[id] + if wasPersisted && now.Sub(persisted) < selfServiceHeartbeatStampInterval { + return nil + } + if err := st.saveLocked(); err != nil { + return err + } + st.stampPersisted[id] = now.UTC() + return nil +} diff --git a/src/pkg/hub/selfservice_registry_test.go b/src/pkg/hub/selfservice_registry_test.go new file mode 100644 index 0000000000..695b0216d2 --- /dev/null +++ b/src/pkg/hub/selfservice_registry_test.go @@ -0,0 +1,341 @@ +package hub + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// newSelfServiceTestStore returns a store on a temp file with a controllable +// clock. +func newSelfServiceTestStore(t *testing.T) (*selfServiceStore, *time.Time) { + t.Helper() + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + st := newSelfServiceStore(filepath.Join(t.TempDir(), "hub-selfservice.json")) + st.now = func() time.Time { return now } + return st, &now +} + +func TestSelfServiceStoreRegisterRoundTrip(t *testing.T) { + st, _ := newSelfServiceTestStore(t) + raw, err := st.register(SelfServiceHive{ID: "byo-alice-lab", Owner: "alice", Org: "alice", Repos: []string{"alice/lab"}}) + if err != nil { + t.Fatalf("register: %v", err) + } + if raw == "" { + t.Fatal("register returned an empty key") + } + if !st.verify("byo-alice-lab", raw) { + t.Fatal("fresh key does not verify") + } + + reloaded := newSelfServiceStore(st.path) + if err := reloaded.load(); err != nil { + t.Fatalf("load: %v", err) + } + h, ok := reloaded.get("byo-alice-lab") + if !ok { + t.Fatal("record lost across reload") + } + if h.Owner != "alice" || len(h.Repos) != 1 || h.KeyHash != hashSelfServiceKey(raw) { + t.Fatalf("record mangled across reload: %+v", h) + } + if !reloaded.verify("byo-alice-lab", raw) { + t.Fatal("key does not verify after reload") + } +} + +func TestSelfServiceStoreRefusesBadInput(t *testing.T) { + st, _ := newSelfServiceTestStore(t) + if _, err := st.register(SelfServiceHive{ID: "hosted-alice", Owner: "alice"}); err != errSelfServiceNotByoID { + t.Fatalf("non-byo ID accepted: err=%v", err) + } + if _, err := st.register(SelfServiceHive{ID: "byo-alice-lab"}); err != errSelfServiceEmptyOwner { + t.Fatalf("empty owner accepted: err=%v", err) + } + if _, err := st.register(SelfServiceHive{ID: "byo-alice-lab", Owner: "alice"}); err != nil { + t.Fatalf("register: %v", err) + } + if _, err := st.register(SelfServiceHive{ID: "BYO-Alice-Lab", Owner: "mallory"}); err != errSelfServiceExists { + t.Fatalf("duplicate (case-folded) ID accepted: err=%v", err) + } + if h, _ := st.get("byo-alice-lab"); h.Owner != "alice" { + t.Fatalf("duplicate registration overwrote the owner: %+v", h) + } + var nilStore *selfServiceStore + if _, err := nilStore.register(SelfServiceHive{ID: "byo-x", Owner: "x"}); err != errSelfServiceNilStore { + t.Fatalf("nil store register err=%v", err) + } + if nilStore.verify("byo-x", "anything") { + t.Fatal("nil store verified a key") + } +} + +func TestSelfServiceStoreRotationOverlap(t *testing.T) { + st, now := newSelfServiceTestStore(t) + old, err := st.register(SelfServiceHive{ID: "byo-alice-lab", Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + fresh, err := st.rotate("byo-alice-lab") + if err != nil { + t.Fatalf("rotate: %v", err) + } + if fresh == old { + t.Fatal("rotate returned the same key") + } + if !st.verify("byo-alice-lab", fresh) || !st.verify("byo-alice-lab", old) { + t.Fatal("inside the overlap window both keys must verify") + } + *now = now.Add(selfServiceKeyRotationOverlap - time.Second) + if !st.verify("byo-alice-lab", old) { + t.Fatal("old key rejected before the overlap window ended") + } + *now = now.Add(2 * time.Second) + if st.verify("byo-alice-lab", old) { + t.Fatal("old key still verifies after the overlap window") + } + if !st.verify("byo-alice-lab", fresh) { + t.Fatal("new key stopped verifying") + } + if _, err := st.rotate("byo-nobody"); err != errSelfServiceNotFound { + t.Fatalf("rotate unknown: err=%v", err) + } +} + +func TestSelfServiceStoreRevoke(t *testing.T) { + st, _ := newSelfServiceTestStore(t) + raw, err := st.register(SelfServiceHive{ID: "byo-alice-lab", Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + removed, err := st.revoke("byo-alice-lab") + if err != nil || !removed { + t.Fatalf("revoke: removed=%v err=%v", removed, err) + } + if st.verify("byo-alice-lab", raw) { + t.Fatal("revoked key still verifies") + } + if removed, _ := st.revoke("byo-alice-lab"); removed { + t.Fatal("second revoke reported a removal") + } + reloaded := newSelfServiceStore(st.path) + if err := reloaded.load(); err != nil { + t.Fatalf("load: %v", err) + } + if len(reloaded.list()) != 0 { + t.Fatal("revoked entry survived on disk") + } +} + +// TestSelfServiceStoreNeverPersistsRawKey is the invariant the design rests on: +// leaked hub state must not let anyone reconstruct a key. +func TestSelfServiceStoreNeverPersistsRawKey(t *testing.T) { + st, _ := newSelfServiceTestStore(t) + raw, err := st.register(SelfServiceHive{ID: "byo-alice-lab", Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + rotated, err := st.rotate("byo-alice-lab") + if err != nil { + t.Fatalf("rotate: %v", err) + } + data, err := os.ReadFile(st.path) + if err != nil { + t.Fatalf("read store: %v", err) + } + for name, key := range map[string]string{"registered": raw, "rotated": rotated} { + if strings.Contains(string(data), key) { + t.Fatalf("%s raw key persisted to disk", name) + } + } + if !strings.Contains(string(data), hashSelfServiceKey(rotated)) { + t.Fatal("store does not hold the hash of the current key") + } + info, err := os.Stat(st.path) + if err != nil { + t.Fatal(err) + } + if perm := info.Mode().Perm(); perm != selfServiceStoreFileMode { + t.Fatalf("store file mode = %o, want %o", perm, selfServiceStoreFileMode) + } +} + +// newSelfServiceHeartbeatHub is newHeartbeatHub with a master secret set (so +// the bearer check runs), no master generations (single-master path), a +// self-service store on a temp file and a captured logger. +func newSelfServiceHeartbeatHub(t *testing.T) (*HubServer, *bytes.Buffer) { + t.Helper() + s := newHeartbeatHub() + s.hubSecret = testHubSecret + s.selfService = newSelfServiceStore(filepath.Join(t.TempDir(), "hub-selfservice.json")) + var buf bytes.Buffer + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + return s, &buf +} + +func postHeartbeatWithBearer(t *testing.T, s *HubServer, hiveID, bearer string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/api/heartbeat", strings.NewReader(`{"hive_id":"`+hiveID+`","org":"alice"}`)) + req.Header.Set("Content-Type", "application/json") + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + rec := httptest.NewRecorder() + s.handleHeartbeat(rec, req) + return rec +} + +// TestSelfServiceHeartbeatBearerIsBoundToOneHive covers the four F2 invariants +// for the byo- lane through the real handler. +func TestSelfServiceHeartbeatBearerIsBoundToOneHive(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + s, logBuf := newSelfServiceHeartbeatHub(t) + + const id = "byo-alice-lab" + raw, err := s.selfService.register(SelfServiceHive{ID: id, Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + if _, err := s.selfService.register(SelfServiceHive{ID: "byo-bob-lab", Owner: "bob"}); err != nil { + t.Fatalf("register bob: %v", err) + } + + if rec := postHeartbeatWithBearer(t, s, id, raw); rec.Code != http.StatusOK { + t.Fatalf("stored key: status=%d body=%s", rec.Code, rec.Body.String()) + } + if rec := postHeartbeatWithBearer(t, s, id, ""); rec.Code != http.StatusUnauthorized { + t.Fatalf("no bearer: status=%d, want 401", rec.Code) + } + masterDerived := derivePerHiveKey(s.hubSecret, infoHeartbeatKey, id) + if rec := postHeartbeatWithBearer(t, s, id, masterDerived); rec.Code != http.StatusUnauthorized { + t.Fatalf("master-derived key authenticated a byo- hive: status=%d", rec.Code) + } + if rec := postHeartbeatWithBearer(t, s, id, s.hubSecret); rec.Code != http.StatusUnauthorized { + t.Fatalf("raw master authenticated a byo- hive: status=%d", rec.Code) + } + if rec := postHeartbeatWithBearer(t, s, "byo-bob-lab", raw); rec.Code != http.StatusUnauthorized { + t.Fatalf("alice's key authenticated bob's byo- hive: status=%d", rec.Code) + } + if rec := postHeartbeatWithBearer(t, s, "alice-lab", raw); rec.Code != http.StatusUnauthorized { + t.Fatalf("self-service key authenticated an operator-listed hive: status=%d", rec.Code) + } + if rec := postHeartbeatWithBearer(t, s, "hosted-alice-lab", raw); rec.Code != http.StatusUnauthorized { + t.Fatalf("self-service key authenticated a hosted hive: status=%d", rec.Code) + } + // The operator-listed lane is untouched: a master-derived key still works + // for a non-byo ID. + if rec := postHeartbeatWithBearer(t, s, "alice-lab", derivePerHiveKey(s.hubSecret, infoHeartbeatKey, "alice-lab")); rec.Code != http.StatusOK { + t.Fatalf("master-derived key for an operator-listed hive: status=%d body=%s", rec.Code, rec.Body.String()) + } + + if strings.Contains(logBuf.String(), raw) { + t.Fatal("raw self-service key appeared in hub logs") + } + if strings.Contains(logBuf.String(), s.hubSecret) { + t.Fatal("master secret appeared in hub logs") + } +} + +func TestSelfServiceHeartbeatResponseCarriesNoKeyMaterial(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + s, _ := newSelfServiceHeartbeatHub(t) + const id = "byo-alice-lab" + raw, err := s.selfService.register(SelfServiceHive{ID: id, Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + s.pendingGitHubAppConfigs[id] = &HeartbeatGitHubAppConfig{AppID: 42, InstallationID: 7, PrivateKey: "-----BEGIN KEY-----"} + rec := postHeartbeatWithBearer(t, s, id, raw) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for name, needle := range map[string]string{ + "raw key": raw, + "key hash": hashSelfServiceKey(raw), + "master secret": s.hubSecret, + "App key": "BEGIN KEY", + } { + if strings.Contains(body, needle) { + t.Fatalf("%s appeared in the heartbeat response", name) + } + } +} + +func TestSelfServiceHeartbeatStampsAcceptedBeatOnly(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + s, _ := newSelfServiceHeartbeatHub(t) + const id = "byo-alice-lab" + raw, err := s.selfService.register(SelfServiceHive{ID: id, Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + if rec := postHeartbeatWithBearer(t, s, id, "wrong-key"); rec.Code != http.StatusUnauthorized { + t.Fatalf("wrong key: status=%d", rec.Code) + } + if h, _ := s.selfService.get(id); !h.LastAcceptedHeartbeatAt.IsZero() { + t.Fatal("a rejected heartbeat stamped LastAcceptedHeartbeatAt") + } + if rec := postHeartbeatWithBearer(t, s, id, raw); rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + h, _ := s.selfService.get(id) + if h.LastAcceptedHeartbeatAt.IsZero() { + t.Fatal("accepted heartbeat did not stamp LastAcceptedHeartbeatAt") + } + reloaded := newSelfServiceStore(s.selfService.path) + if err := reloaded.load(); err != nil { + t.Fatalf("load: %v", err) + } + if rh, _ := reloaded.get(id); rh.LastAcceptedHeartbeatAt.IsZero() { + t.Fatal("first accepted heartbeat stamp was not persisted") + } +} + +func TestSelfServiceStoreStampPersistIsDebounced(t *testing.T) { + st, now := newSelfServiceTestStore(t) + if _, err := st.register(SelfServiceHive{ID: "byo-alice-lab", Owner: "alice"}); err != nil { + t.Fatalf("register: %v", err) + } + first := *now + if err := st.noteAcceptedHeartbeat("byo-alice-lab", first); err != nil { + t.Fatal(err) + } + second := first.Add(selfServiceHeartbeatStampInterval / 2) + if err := st.noteAcceptedHeartbeat("byo-alice-lab", second); err != nil { + t.Fatal(err) + } + onDisk := newSelfServiceStore(st.path) + if err := onDisk.load(); err != nil { + t.Fatal(err) + } + if h, _ := onDisk.get("byo-alice-lab"); !h.LastAcceptedHeartbeatAt.Equal(first) { + t.Fatalf("stamp inside the interval was persisted: disk=%v want %v", h.LastAcceptedHeartbeatAt, first) + } + if h, _ := st.get("byo-alice-lab"); !h.LastAcceptedHeartbeatAt.Equal(second) { + t.Fatalf("in-memory stamp not current: %v", h.LastAcceptedHeartbeatAt) + } + third := first.Add(selfServiceHeartbeatStampInterval + time.Second) + if err := st.noteAcceptedHeartbeat("byo-alice-lab", third); err != nil { + t.Fatal(err) + } + if err := onDisk.load(); err != nil { + t.Fatal(err) + } + if h, _ := onDisk.get("byo-alice-lab"); !h.LastAcceptedHeartbeatAt.Equal(third) { + t.Fatalf("stamp past the interval was not persisted: disk=%v want %v", h.LastAcceptedHeartbeatAt, third) + } + if err := st.noteAcceptedHeartbeat("byo-unknown", third); err != nil { + t.Fatalf("unknown ID stamp returned an error: %v", err) + } +} diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 5638807292..b0d696d560 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1045,8 +1045,12 @@ type HubServer struct { hubGitBranch string sharedAgentPool *sharedAgentPool hubSecret string - myHivesCacheMu sync.Mutex - myHivesCache map[string]myHivesSnapshot + // selfService holds owner-registered (byo-) hives and the hashes of their + // random heartbeat keys (#18060). Nil on bare test literals; every method + // is nil-safe and answers "nothing registered". + selfService *selfServiceStore + myHivesCacheMu sync.Mutex + myHivesCache map[string]myHivesSnapshot // keyGenerations is the ordered set of master generations this hub accepts // (hub_generations.go). Until a rotation happens it holds exactly ONE // generation whose secret IS hubSecret, so every derived key is @@ -1575,6 +1579,7 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub saveLoopStop: make(chan struct{}), saveLoopDone: make(chan struct{}), registryPath: registryPath, + selfService: newSelfServiceStore(selfServiceStorePath), hubGitHash: gitHash, hubGitBranch: gitBranch, sharedAgentPool: newSharedAgentPool(), @@ -1701,6 +1706,13 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub } s.loadRegistry() + if err := s.selfService.load(); err != nil { + // Fail closed: an unreadable store means no self-service hive + // authenticates until an operator fixes the file. + logger.Warn("self-service registry not loaded; byo- hives will not authenticate", "path", selfServiceStorePath, "error", err) + } else if n := len(s.selfService.list()); n > 0 { + logger.Info("self-service registry loaded", "hives", n) + } logHubGitHubIdentityMode(logger) // Rebuild the in-memory upgrade-delivery map from the durable registry // BEFORE the server takes its first heartbeat, so a hub restart can never @@ -1930,6 +1942,14 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { // retained to back GET /api/saas/admin/auth-rollout, where an operator can // confirm no hive regressed after the cutover. s.noteHeartbeatAuthPath(payload.HiveID, s.heartbeatBearerIsPerHive(presentedBearer, payload.HiveID)) + // A self-service hive's accepted heartbeat is the liveness signal its + // registration expiry reads (#18060). Stamped only after the bearer + // bound this beat to the claimed ID. + if IsSelfServiceHiveID(payload.HiveID) { + if err := s.selfService.noteAcceptedHeartbeat(payload.HiveID, time.Now()); err != nil { + s.logger.Warn("self-service heartbeat stamp not persisted", "hive", payload.HiveID, "error", err) + } + } } // Adopt the spoke's reported dashboard-token hash into the hub's stored // record — the reference spoke-relayed upgrade requests verify their proof From ba56cc7b1fff2bf6fdd3b03e96f68a32a1714dfe Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 19:14:13 -0500 Subject: [PATCH 190/223] =?UTF-8?q?=F0=9F=90=9B=20fix(agent):=20permission?= =?UTF-8?q?s=20watcher=20skips=20chowns=20it=20can=20never=20perform=20ins?= =?UTF-8?q?tead=20of=20warning=20per=20path=20(#18143)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unprivileged watcher (dev, uid 1001, no CAP_CHOWN after the entrypoint's privilege drop) can never chown an inode another uid owns, so every such entry failed with EPERM and the per-path deduper warned once for each of them. A surge of sandbox kicks writes thousands of paths under /data/agents/sandbox/<slug>-<nonce>/.claude/skills/... as the Job pod's uid, which produced 4087 chown-failed WARN lines in ten minutes before the 4096-path dedupe cap saturated (#18120). Decide the chown is unrepairable before the syscall (euid != 0 and euid != owner), leave the entry's mode alone as not ours, and report all such entries as one summary line per tick: WARN when the condition appears or its count grows, DEBUG while unchanged, nothing once it clears. Entries the process owns are repaired exactly as before. Fixes #18120 Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...d-18120-permissions-watcher-chown-eperm.md | 1 + src/crux/agent/permissions_unownable.go | 91 +++++++ src/crux/agent/permissions_unownable_test.go | 227 ++++++++++++++++++ src/crux/agent/permissions_watcher.go | 11 + 4 files changed, 330 insertions(+) create mode 100644 changelog.d/fixed-18120-permissions-watcher-chown-eperm.md create mode 100644 src/crux/agent/permissions_unownable.go create mode 100644 src/crux/agent/permissions_unownable_test.go diff --git a/changelog.d/fixed-18120-permissions-watcher-chown-eperm.md b/changelog.d/fixed-18120-permissions-watcher-chown-eperm.md new file mode 100644 index 0000000000..2569b441f4 --- /dev/null +++ b/changelog.d/fixed-18120-permissions-watcher-chown-eperm.md @@ -0,0 +1 @@ +- The permissions watcher no longer attempts, or logs per path, a chown it can never perform. After the entrypoint's privilege drop the hive process runs as `dev` (uid 1001) without `CAP_CHOWN`, so an entry another uid owns, such as a sandbox kick workspace under `/data/agents/sandbox/<slug>-<nonce>/.claude/skills/...` written by the Job pod's uid, always fails with `operation not permitted`; a surge of kicks minted ~4096 such paths between two ticks and the per-path deduper warned once for each of them (4087 `chown failed` WARN lines in ten minutes) before saturating. The watcher now decides ownership is unrepairable before the syscall, leaves the entry's mode alone as not ours, and reports all such entries as one summary line per tick: WARN with the count, a sample path and a hint when the condition appears or grows, DEBUG while it holds unchanged, nothing once it clears. Entries the process owns are still chowned and chmodded exactly as before (#18120). diff --git a/src/crux/agent/permissions_unownable.go b/src/crux/agent/permissions_unownable.go new file mode 100644 index 0000000000..a575b545bf --- /dev/null +++ b/src/crux/agent/permissions_unownable.go @@ -0,0 +1,91 @@ +package agent + +import ( + "log/slog" + "os" + "sync" +) + +// unownableHint tells the operator who can actually repair the ownership the +// watcher is reporting. +const unownableHint = "chown needs root (CAP_CHOWN); the watcher runs unprivileged after the entrypoint's privilege drop, so these entries are repaired by the entrypoint's root phase or out of band, never by retrying here" + +// cannotChown reports whether this process certainly cannot chown an entry +// owned by ownerUID, so attempting it would only ever return EPERM. +// +// chown(2) lets an unprivileged caller change only the GROUP of an inode it +// OWNS (to a group it belongs to); changing the uid, or anything about an +// inode owned by someone else, needs CAP_CHOWN. The hive process drops to dev +// (uid 1001) at the entrypoint, so every entry another uid owns — a root seed, +// or a sandbox kick's workspace under /data/agents/sandbox/<slug>-<nonce> +// written by the Job pod's uid (#18120) — is unrepairable by construction, and +// a surge of kicks mints thousands of such paths between two ticks. Deciding +// that here, before the syscall, is what keeps the failure from being logged +// once per path (4087 WARN lines in ten minutes on hive-tall-frog). +func cannotChown(ownerUID uint32) bool { + euid := os.Geteuid() + return euid != 0 && uint32(euid) != ownerUID +} + +// unownableTally counts, per watcher tick, the entries fixEntry skipped +// because cannotChown held, and reports them as ONE summary line per tick +// instead of one line per path. The summary is WARN only when the count sets +// a new high-water mark since the condition last cleared, and DEBUG otherwise, +// so a standing unfixable tree costs one WARN when it appears, one more each +// time it grows, and nothing at WARN in steady state. +type unownableTally struct { + mu sync.Mutex + count int // entries skipped this tick + sample string // first skipped path this tick, for the operator's grep + highWater int // largest count already reported at WARN; 0 once clear +} + +// skipped records one entry the watcher could not own this tick. +func (u *unownableTally) skipped(path string) { + u.mu.Lock() + defer u.mu.Unlock() + if u.count == 0 { + u.sample = path + } + u.count++ +} + +// flush ends the tick: emits the summary and resets the per-tick counters. It +// returns whether the summary went out at WARN (tests). +func (u *unownableTally) flush(logger *slog.Logger) bool { + u.mu.Lock() + count, sample := u.count, u.sample + u.count, u.sample = 0, "" + if count == 0 { + // Condition cleared: the next appearance is a new finding. + u.highWater = 0 + u.mu.Unlock() + return false + } + warn := count > u.highWater + if warn { + u.highWater = count + } + u.mu.Unlock() + + attrs := []any{ + "count", count, + "sample_path", sample, + "watcher_uid", os.Geteuid(), + "hint", unownableHint, + } + if warn { + logger.Warn("permissions watcher: entries owned by another uid need a chown this process cannot perform; skipped without retrying", + append(attrs, "note", "one summary per tick; logged at debug until the count grows or the condition clears")...) + return true + } + logger.Debug("permissions watcher: entries owned by another uid skipped (unchanged)", attrs...) + return false +} + +// permUnownable is the watcher's shared tally. Package-level for the same +// reason as permWarnDedupe: fixEntry keeps its (path, fi, logger) signature. +var permUnownable = &unownableTally{} + +// resetPermUnownable restores pristine tally state. Tests only. +func resetPermUnownable() { permUnownable = &unownableTally{} } diff --git a/src/crux/agent/permissions_unownable_test.go b/src/crux/agent/permissions_unownable_test.go new file mode 100644 index 0000000000..bba2262741 --- /dev/null +++ b/src/crux/agent/permissions_unownable_test.go @@ -0,0 +1,227 @@ +package agent + +import ( + "log/slog" + "os" + "path/filepath" + "strconv" + "testing" +) + +// These tests pin the #18120 fix: an entry owned by a uid the unprivileged +// watcher can never chown (a sandbox kick workspace written by the Job pod's +// uid, a root seed) is skipped BEFORE the syscall and reported as one summary +// line per tick, never as one WARN per path. On hive-tall-frog a surge of +// kicks minted ~4096 such paths and the per-path deduper warned once for each +// of them before saturating. + +const unownableSummaryMsg = "permissions watcher: entries owned by another uid need a chown this process cannot perform; skipped without retrying" + +// foreignOwnedEntry creates a regular file and returns a FileInfo that reports +// it as owned by a uid this process is not, with a gid that is not NodeGID, so +// fixEntry wants a chown it cannot perform. Skips as root, where every chown +// is possible and the skip arm is unreachable. +func foreignOwnedEntry(t *testing.T, dir, name string) (string, os.FileInfo) { + t.Helper() + if os.Geteuid() == 0 { + t.Skip("running as root: every chown is permitted, skip arm unreachable") + } + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("{}"), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + fi, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + foreign := uint32(os.Geteuid() + 1) + return path, fakeOwnerInfo{FileInfo: fi, sys: syntheticStat(t, path, foreign, uint32(NodeGID)+1)} +} + +func TestCannotChownUnprivilegedForeignOwner(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: chown is always permitted") + } + me := uint32(os.Geteuid()) + if cannotChown(me) { + t.Error("an entry this process owns must be attempted (group change may succeed)") + } + if !cannotChown(me + 1) { + t.Error("an entry another uid owns can never be chowned without CAP_CHOWN") + } + if !cannotChown(0) { + t.Error("a root-owned entry can never be chowned without CAP_CHOWN") + } +} + +// TestFixEntryForeignOwnerSkippedWithoutPerPathWarn is the core regression +// test: many foreign-owned entries across several ticks produce ZERO per-path +// chown WARN lines and leave the mode untouched (0600 stays 0600: the entry is +// not ours to widen either). +func TestFixEntryForeignOwnerSkippedWithoutPerPathWarn(t *testing.T) { + resetPermWarnDedupe() + resetPermUnownable() + t.Cleanup(resetPermWarnDedupe) + t.Cleanup(resetPermUnownable) + + dir := t.TempDir() + const paths = 50 + h := &recordingHandler{} + logger := slog.New(h) + var first string + for i := range paths { + path, fi := foreignOwnedEntry(t, dir, "skill-"+strconv.Itoa(i)) + if i == 0 { + first = path + } + fixEntry(path, fi, logger) + } + + if got := h.count(slog.LevelWarn, "permissions watcher: chown failed", ""); got != 0 { + t.Errorf("per-path chown WARN logged %d times for unownable entries, want 0", got) + } + if got := h.count(slog.LevelDebug, "permissions watcher: chown failed", ""); got != 0 { + t.Errorf("per-path chown DEBUG logged %d times: the chown must not be attempted at all", got) + } + if got := h.count(slog.LevelWarn, "permissions watcher: chmod file failed", ""); got != 0 { + t.Errorf("chmod attempted on %d entries we do not own, want 0", got) + } + info, err := os.Stat(first) + if err != nil { + t.Fatalf("stat: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("mode of foreign-owned entry changed to %v, want 0600 untouched", info.Mode().Perm()) + } + + permUnownable.mu.Lock() + count, sample := permUnownable.count, permUnownable.sample + permUnownable.mu.Unlock() + if count != paths { + t.Errorf("tally counted %d skipped entries, want %d", count, paths) + } + if sample != first { + t.Errorf("tally sample = %q, want the first skipped path %q", sample, first) + } +} + +// TestUnownableTallyOneSummaryPerTickAtMost pins the summary cadence: WARN +// when the condition appears, DEBUG while it holds unchanged, WARN again only +// when it grows, nothing when the tick saw none, and WARN once more when the +// condition clears and then reappears. +func TestUnownableTallyOneSummaryPerTickAtMost(t *testing.T) { + u := &unownableTally{} + h := &recordingHandler{} + logger := slog.New(h) + + tick := func(n int) bool { + for i := range n { + u.skipped("/data/agents/sandbox/quality-1/.claude/skills/" + strconv.Itoa(i)) + } + return u.flush(logger) + } + + if tick(0) { + t.Error("an empty tick must not warn") + } + if !tick(4087) { + t.Error("first tick with unownable entries must warn") + } + if tick(4087) { + t.Error("unchanged count on the next tick must be demoted to debug") + } + if tick(10) { + t.Error("a smaller count must not warn") + } + if !tick(5000) { + t.Error("a count above the high-water mark is new information and must warn") + } + if tick(0) { + t.Error("a clear tick must not warn") + } + if !tick(1) { + t.Error("reappearance after the condition cleared must warn again") + } + + if got := h.count(slog.LevelWarn, unownableSummaryMsg, ""); got != 3 { + t.Errorf("summary WARN logged %d times across 7 ticks, want 3", got) + } + if got := h.count(slog.LevelDebug, "permissions watcher: entries owned by another uid skipped (unchanged)", ""); got != 2 { + t.Errorf("summary DEBUG logged %d times, want 2", got) + } + u.mu.Lock() + defer u.mu.Unlock() + if u.count != 0 || u.sample != "" { + t.Errorf("flush left count=%d sample=%q, want reset", u.count, u.sample) + } + if u.highWater != 1 { + t.Errorf("highWater = %d after the final tick of 1, want 1", u.highWater) + } +} + +// TestUnownableTallySummaryCarriesCountAndSample pins the attrs an operator +// needs to act on the one line they get. +func TestUnownableTallySummaryCarriesCountAndSample(t *testing.T) { + u := &unownableTally{} + h := &recordingHandler{} + u.skipped("/data/agents/sandbox/quality-1/.claude/skills/upgrade-vllm") + u.skipped("/data/agents/sandbox/quality-1/.claude/skills/other") + u.flush(slog.New(h)) + + h.mu.Lock() + defer h.mu.Unlock() + if len(h.records) != 1 { + t.Fatalf("got %d records, want 1", len(h.records)) + } + attrs := map[string]string{} + h.records[0].Attrs(func(a slog.Attr) bool { + attrs[a.Key] = a.Value.String() + return true + }) + if attrs["count"] != "2" { + t.Errorf("count attr = %q, want 2", attrs["count"]) + } + if attrs["sample_path"] != "/data/agents/sandbox/quality-1/.claude/skills/upgrade-vllm" { + t.Errorf("sample_path attr = %q, want the first skipped path", attrs["sample_path"]) + } + if attrs["hint"] != unownableHint { + t.Errorf("hint attr = %q, want unownableHint", attrs["hint"]) + } +} + +// TestFixPermissionsFlushesUnownableSummary pins that a watcher tick ends by +// emitting the summary, so the tally does not accumulate across ticks. +func TestFixPermissionsFlushesUnownableSummary(t *testing.T) { + resetPermWarnDedupe() + resetPermUnownable() + t.Cleanup(resetPermWarnDedupe) + t.Cleanup(resetPermUnownable) + + h := &recordingHandler{} + logger := slog.New(h) + // Seed the tally as fixEntry would during the walk; the walk roots are + // left at their production values, which do not exist here, so the tick + // itself touches nothing and only the flush is observable. + path, fi := foreignOwnedEntry(t, t.TempDir(), "upgrade-vllm") + fixEntry(path, fi, logger) + + oldDirs, oldGlob, oldCaps, oldRepo := WatchedHomeDirs, ModeFileGlob, CapsFileGlob, SharedRepoParent + WatchedHomeDirs = nil + ModeFileGlob = filepath.Join(t.TempDir(), "none-*") + CapsFileGlob = filepath.Join(t.TempDir(), "none-*") + SharedRepoParent = t.TempDir() + t.Cleanup(func() { + WatchedHomeDirs, ModeFileGlob, CapsFileGlob, SharedRepoParent = oldDirs, oldGlob, oldCaps, oldRepo + }) + + fixPermissions(logger) + if got := h.count(slog.LevelWarn, unownableSummaryMsg, ""); got != 1 { + t.Errorf("summary WARN logged %d times after one tick, want 1", got) + } + permUnownable.mu.Lock() + count := permUnownable.count + permUnownable.mu.Unlock() + if count != 0 { + t.Errorf("tally count = %d after the tick flushed, want 0", count) + } +} diff --git a/src/crux/agent/permissions_watcher.go b/src/crux/agent/permissions_watcher.go index 648b60fb3d..9615ce7cd4 100644 --- a/src/crux/agent/permissions_watcher.go +++ b/src/crux/agent/permissions_watcher.go @@ -516,6 +516,7 @@ func fixPermissions(logger *slog.Logger) { ) } } + permUnownable.flush(logger) } // fixEntry checks a single file or directory and corrects ownership/mode @@ -555,6 +556,16 @@ func fixEntry(path string, fi os.FileInfo, logger *slog.Logger) { if stat.Gid != uint32(NodeGID) { needsChown = true } + if needsChown && cannotChown(stat.Uid) { + // Unrepairable by construction (#18120): an unprivileged watcher can + // never chown an inode it does not own, so do not spend a syscall and + // a WARN line per path on it. Tally it for the per-tick summary and + // leave the uid alone so the mode arms below skip it as not ours. + permUnownable.skipped(path) + permWarnDedupe.clear(dedupeKey("chown", path)) + needsChown = false + newUID = int(stat.Uid) + } if needsChown { if err := entryChown(f, newUID, NodeGID); err != nil { warnDeduped(logger, "chown", "permissions watcher: chown failed", path, err) From 15cf38e89f995be004fd16d9536e617e2f82912b Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 19:21:45 -0500 Subject: [PATCH 191/223] =?UTF-8?q?=F0=9F=90=9B=20knowledge=20advisor:=20h?= =?UTF-8?q?onour=20Retry-After,=20pace=20hive-wide,=20record=20429=20as=20?= =?UTF-8?q?rate-limited=20without=20spending=20budget=20(#18152)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 knowledge advisor: honour Retry-After, pace hive-wide, record 429 as rate-limited without spending budget The LLM knowledge advisor's deliver arm on hive's hive was starved by 429s from the Copilot endpoint: the advisor spends the same user token as every copilot agent, holdout kicks call the model too, and N agents entering scopes in the same second fanned out N calls against one quota with no Retry-After handling. Every 429 was recorded as error and charged to the per-agent-hour and per-day budgets, so a throttled hour also became an over-budget hour. - Retry a 429 once after Retry-After (seconds or HTTP date), or a jittered 2-4s backoff when absent, only when at least 3s of timeout_s remains. - Record the outcome as rate-limited (new outcome value, same record schema) and refund the budget take; the retro lane counts it in AdvisorRateLimited and assigns no arm. - Hive-wide in-flight cap of one advisor call behind the per-agent guard. - First 429 per five-minute window at WARN with retry-after, x-ratelimit-* headers and body; the rest at DEBUG. Fixes #18151 Refs #17054 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 knowledge advisor: refund the fleet-day counter only when an event was removed Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🌱 knowledge advisor: fix refund test expectation after the post-refund take Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18151-knowledge-advisor-429.md | 1 + src/cmd/hive/knowledge_adviser_llm.go | 285 ++++++++++++++- src/cmd/hive/knowledge_adviser_llm_test.go | 330 +++++++++++++++++- src/docs/agent-knowledge-and-memory.md | 45 ++- src/docs/retro-lane.md | 4 +- src/pkg/retro/analysis.go | 4 +- src/pkg/retro/retro.go | 14 +- src/pkg/retro/retro_test.go | 20 +- 8 files changed, 655 insertions(+), 48 deletions(-) create mode 100644 changelog.d/fixed-18151-knowledge-advisor-429.md diff --git a/changelog.d/fixed-18151-knowledge-advisor-429.md b/changelog.d/fixed-18151-knowledge-advisor-429.md new file mode 100644 index 0000000000..82837471cf --- /dev/null +++ b/changelog.d/fixed-18151-knowledge-advisor-429.md @@ -0,0 +1 @@ +- The model-ranked knowledge advisor no longer burns its deliver arm on 429s from the model endpoint: a 429 is retried once after the upstream's `Retry-After` (a jittered 2 to 4 s backoff when absent) inside the existing `timeout_s`, a decision that still ends throttled is recorded as the new `rate-limited` outcome (distinct from `error`, excluded from the retro arm comparison) and is not charged against the per-agent-hour or per-day budgets, at most one advisor call is in flight across the whole hive on top of the per-agent guard, and the first 429 in each five-minute window logs at WARN with the `Retry-After`, `x-ratelimit-*` headers and response body while the rest log at DEBUG (#18151). diff --git a/src/cmd/hive/knowledge_adviser_llm.go b/src/cmd/hive/knowledge_adviser_llm.go index 233a540260..c711084250 100644 --- a/src/cmd/hive/knowledge_adviser_llm.go +++ b/src/cmd/hive/knowledge_adviser_llm.go @@ -9,10 +9,15 @@ import ( "hash/fnv" "io" "log/slog" + "maps" + "math/rand/v2" "net" "net/http" + "slices" + "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/hivecommons/hive/crux" @@ -39,16 +44,40 @@ const ( // knowledgeAdviserLLMBudgetWindow is the rolling per-agent window. knowledgeAdviserLLMBudgetWindow = time.Hour knowledgeAdviserLLMUserPrefix = "hive-knowledge-advisor:" + + // knowledgeAdviserLLMMaxInFlight caps model calls in flight across every + // agent on the hive (hivecommons/hive#18151). The advisor shares one + // upstream quota with the agents (the Copilot user token), so N agents + // fanning out N calls in the same second only buys 429s; the runtime's + // per-agent in-flight guard stays in front of this cap. + knowledgeAdviserLLMMaxInFlight = 1 + // knowledgeAdviserLLMRetryBackoff is the wait before the single retry of + // a 429 that carries no usable Retry-After, plus up to the same amount of + // jitter so agents released together do not collide again. + knowledgeAdviserLLMRetryBackoff = 2 * time.Second + // knowledgeAdviserLLMRetryHeadroom is how much of the decision timeout + // must remain after the Retry-After wait for the retry to be worth + // making; a shorter remainder records rate-limited without a second call. + knowledgeAdviserLLMRetryHeadroom = 3 * time.Second + // knowledgeAdviserLLMRateLimitWarnWindow: the first 429 in this window + // logs at WARN with the rate-limit headers, the rest at DEBUG. + knowledgeAdviserLLMRateLimitWarnWindow = 5 * time.Minute + // knowledgeAdviserLLMErrorBodyRunes caps the logged 429 response body. + knowledgeAdviserLLMErrorBodyRunes = 300 + // knowledgeAdviserLLMRateLimitHeaderPrefix selects the upstream's + // rate-limit headers for the WARN line. + knowledgeAdviserLLMRateLimitHeaderPrefix = "x-ratelimit-" ) // Model verdicts: what the model call produced. const ( - knowledgeVerdictPick = "pick" - knowledgeVerdictNull = "null" - knowledgeVerdictInvalid = "invalid" - knowledgeVerdictTimeout = "timeout" - knowledgeVerdictOverBudget = "over-budget" - knowledgeVerdictError = "error" + knowledgeVerdictPick = "pick" + knowledgeVerdictNull = "null" + knowledgeVerdictInvalid = "invalid" + knowledgeVerdictTimeout = "timeout" + knowledgeVerdictOverBudget = "over-budget" + knowledgeVerdictRateLimited = "rate-limited" + knowledgeVerdictError = "error" ) // Decision outcomes: what happened to the delivery. invalid, timeout, @@ -83,9 +112,39 @@ type knowledgeAdviserLLM struct { // decisions persists one compact record per decision for the retro // lane's arm comparison (#17211); nil records nothing. decisions knowledgeAdvisorDecisionRecorder + // slots is the hive-wide in-flight cap (knowledgeAdviserLLMMaxInFlight); + // nil means uncapped. + slots chan struct{} + // waiting counts decisions blocked on slots, for tests and the WARN line. + waiting atomic.Int32 + // wait blocks for d or until ctx ends and reports whether d elapsed; tests + // replace it so no test sleeps. + wait func(ctx context.Context, d time.Duration) bool mu sync.Mutex delivered map[string]*knowledgeAdviserLLMKick + // rateLimitWarnedAt is when the current WARN window opened. + rateLimitWarnedAt time.Time +} + +// newKnowledgeAdviserLLMSlots is the hive-wide in-flight semaphore. +func newKnowledgeAdviserLLMSlots() chan struct{} { + return make(chan struct{}, knowledgeAdviserLLMMaxInFlight) +} + +// knowledgeAdviserLLMWait is the production wait: a timer bounded by ctx. +func knowledgeAdviserLLMWait(ctx context.Context, d time.Duration) bool { + if d <= 0 { + return ctx.Err() == nil + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return true + case <-ctx.Done(): + return false + } } // knowledgeAdviserLLMKick is the slugs this decorator handed the runtime @@ -111,6 +170,8 @@ func newKnowledgeAdviserLLM(b *boot, inner *knowledgeAdviser) *knowledgeAdviserL logger: b.logger, budget: newKnowledgeAdviserLLMBudget(time.Now), decisions: b.knowledgeAdvisorDecisions(), + slots: newKnowledgeAdviserLLMSlots(), + wait: knowledgeAdviserLLMWait, delivered: make(map[string]*knowledgeAdviserLLMKick), } } @@ -253,17 +314,36 @@ type knowledgeAdviserLLMDecision struct { latency time.Duration promptTokens int completionTokens int + // retryAfter is the upstream's Retry-After on a rate-limited decision, + // zero when it sent none. + retryAfter time.Duration + // retried reports that the single 429 retry was made. + retried bool } -// decide spends one budgeted call on the model. There is no retry: a wrong -// answer costs one silent scope, a retry would double the spend. +// decide spends one budgeted call on the model. A wrong answer is never +// retried (it would double the spend); a 429 is retried once after the +// upstream's Retry-After, inside the same timeout, and a decision that still +// ends rate-limited hands its budget back (hivecommons/hive#18151). func (d *knowledgeAdviserLLM) decide(agent string, kick scheduler.KickSummary, scope crux.KnowledgeScope, candidates []crux.KnowledgeAdvice, r config.ResolvedAdvisorLLM) knowledgeAdviserLLMDecision { - if over := d.budget.take(agent, r.MaxCallsPerAgentHour, r.MaxCallsPerDay); over != "" { + over, spent := d.budget.take(agent, r.MaxCallsPerAgentHour, r.MaxCallsPerDay) + if over != "" { return knowledgeAdviserLLMDecision{verdict: knowledgeVerdictOverBudget, budget: over} } start := d.now() ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.TimeoutS)*time.Second) defer cancel() + if !d.acquireSlot(ctx) { + // The hive-wide slot never freed inside the timeout: nothing was + // sent, so nothing is charged. + d.budget.refund(agent, spent) + return knowledgeAdviserLLMDecision{ + verdict: knowledgeVerdictTimeout, + err: errors.New("knowledge advisor: timed out waiting for the hive-wide in-flight slot"), + latency: d.now().Sub(start), + } + } + defer d.releaseSlot() reqBody := knowledgeAdviserLLMRequest{ Model: r.Model, Messages: []knowledgeAdviserLLMMessage{ @@ -275,12 +355,33 @@ func (d *knowledgeAdviserLLM) decide(agent string, kick scheduler.KickSummary, s User: knowledgeAdviserLLMUserPrefix + agent, } resp, err := d.complete(ctx, r, reqBody) - dec := knowledgeAdviserLLMDecision{latency: d.now().Sub(start)} + dec := knowledgeAdviserLLMDecision{} + var rl *knowledgeAdviserLLMRateLimited + if errors.As(err, &rl) { + d.logRateLimited(agent, rl, false) + dec.retryAfter = rl.retryAfter + if wait, ok := knowledgeAdviserLLMRetryWait(ctx, rl.retryAfter, time.Now()); ok && d.wait(ctx, wait) { + dec.retried = true + resp, err = d.complete(ctx, r, reqBody) + if errors.As(err, &rl) { + d.logRateLimited(agent, rl, true) + dec.retryAfter = rl.retryAfter + } + } + } + dec.latency = d.now().Sub(start) if err != nil { dec.err = err dec.verdict = knowledgeVerdictError var ne net.Error - if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()) { + switch { + case errors.As(err, &rl): + // Nothing was answered: the call is not spent against the + // budgets, so a throttled hour does not also become an + // over-budget hour. + dec.verdict = knowledgeVerdictRateLimited + d.budget.refund(agent, spent) + case errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()): dec.verdict = knowledgeVerdictTimeout } return dec @@ -375,6 +476,9 @@ func (d *knowledgeAdviserLLM) complete(ctx context.Context, r config.ResolvedAdv return cr, fmt.Errorf("knowledge advisor request failed: %w", err) } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusTooManyRequests { + return cr, newKnowledgeAdviserLLMRateLimited(resp, time.Now()) + } if resp.StatusCode != http.StatusOK { return cr, fmt.Errorf("knowledge advisor model returned %d", resp.StatusCode) } @@ -421,12 +525,131 @@ func (d *knowledgeAdviserLLM) logDecision(agent, kickID string, scope crux.Knowl if dec.budget != "" { attrs = append(attrs, "budget", dec.budget) } + if dec.verdict == knowledgeVerdictRateLimited { + attrs = append(attrs, "retry_after_s", dec.retryAfter.Seconds(), "retried", dec.retried) + } if dec.err != nil { attrs = append(attrs, "error", dec.err.Error()) } d.logger.Info("knowledge advisor: decision", attrs...) } +// knowledgeAdviserLLMRateLimited is a 429 from the model endpoint with what +// the upstream said about it. +type knowledgeAdviserLLMRateLimited struct { + // retryAfter is the parsed Retry-After, zero when absent or malformed. + retryAfter time.Duration + // headers holds retry-after and the x-ratelimit-* headers, lower-cased. + headers map[string]string + // body is the response body, capped at knowledgeAdviserLLMErrorBodyRunes. + body string +} + +func (e *knowledgeAdviserLLMRateLimited) Error() string { + return fmt.Sprintf("knowledge advisor model returned %d", http.StatusTooManyRequests) +} + +// newKnowledgeAdviserLLMRateLimited reads the 429's headers and a bounded +// slice of its body; the body is drained no further. +func newKnowledgeAdviserLLMRateLimited(resp *http.Response, now time.Time) *knowledgeAdviserLLMRateLimited { + e := &knowledgeAdviserLLMRateLimited{headers: map[string]string{}} + for name, values := range resp.Header { + lower := strings.ToLower(name) + if (lower == "retry-after" || strings.HasPrefix(lower, knowledgeAdviserLLMRateLimitHeaderPrefix)) && len(values) > 0 { + e.headers[lower] = values[0] + } + } + e.retryAfter = parseRetryAfter(resp.Header.Get("Retry-After"), now) + raw, _ := io.ReadAll(io.LimitReader(resp.Body, int64(knowledgeAdviserLLMErrorBodyRunes)*4)) + e.body = truncateRunes(strings.Join(strings.Fields(string(raw)), " "), knowledgeAdviserLLMErrorBodyRunes) + return e +} + +// parseRetryAfter reads a Retry-After header: delay seconds or an HTTP date. +// Anything else, or a date already past, is zero. +func parseRetryAfter(v string, now time.Time) time.Duration { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if secs, err := strconv.ParseFloat(v, 64); err == nil { + if secs <= 0 { + return 0 + } + return time.Duration(secs * float64(time.Second)) + } + if at, err := http.ParseTime(v); err == nil && at.After(now) { + return at.Sub(now) + } + return 0 +} + +// knowledgeAdviserLLMRetryWait is how long to wait before the single retry of +// a 429: the Retry-After when the upstream sent one, else the jittered +// backoff. ok is false when that wait would leave less than the retry +// headroom before ctx's deadline, so the retry is skipped and the decision +// records rate-limited without a second call. +func knowledgeAdviserLLMRetryWait(ctx context.Context, retryAfter time.Duration, now time.Time) (time.Duration, bool) { + wait := retryAfter + if wait <= 0 { + wait = knowledgeAdviserLLMRetryBackoff + rand.N(knowledgeAdviserLLMRetryBackoff) + } + if deadline, ok := ctx.Deadline(); ok && now.Add(wait+knowledgeAdviserLLMRetryHeadroom).After(deadline) { + return wait, false + } + return wait, true +} + +// logRateLimited logs a 429: the first in each WARN window at WARN with the +// upstream's rate-limit headers and body, the rest at DEBUG. The decision +// line itself still goes out at INFO. +func (d *knowledgeAdviserLLM) logRateLimited(agent string, rl *knowledgeAdviserLLMRateLimited, retried bool) { + now := d.now() + d.mu.Lock() + first := d.rateLimitWarnedAt.IsZero() || now.Sub(d.rateLimitWarnedAt) >= knowledgeAdviserLLMRateLimitWarnWindow + if first { + d.rateLimitWarnedAt = now + } + d.mu.Unlock() + attrs := []any{ + "agent", agent, + "retry_after_s", rl.retryAfter.Seconds(), + "retried", retried, + "waiting", d.waiting.Load(), + } + if first { + for _, k := range slices.Sorted(maps.Keys(rl.headers)) { + attrs = append(attrs, "header_"+strings.ReplaceAll(k, "-", "_"), rl.headers[k]) + } + attrs = append(attrs, "body", rl.body, "warn_window", knowledgeAdviserLLMRateLimitWarnWindow.String()) + d.logger.Warn("knowledge advisor: model endpoint rate limited (429)", attrs...) + return + } + d.logger.Debug("knowledge advisor: model endpoint rate limited (429)", attrs...) +} + +// acquireSlot takes the hive-wide in-flight slot, or reports false when ctx +// ends first. A nil slots channel is uncapped. +func (d *knowledgeAdviserLLM) acquireSlot(ctx context.Context) bool { + if d.slots == nil { + return true + } + d.waiting.Add(1) + defer d.waiting.Add(-1) + select { + case d.slots <- struct{}{}: + return true + case <-ctx.Done(): + return false + } +} + +func (d *knowledgeAdviserLLM) releaseSlot() { + if d.slots != nil { + <-d.slots + } +} + type knowledgeAdviserLLMMessage struct { Role string `json:"role"` Content string `json:"content"` @@ -470,9 +693,10 @@ func newKnowledgeAdviserLLMBudget(now func() time.Time) *knowledgeAdviserLLMBudg return &knowledgeAdviserLLMBudget{now: now, events: make(map[string][]time.Time)} } -// take spends one call for agent when both budgets allow it and returns ""; -// otherwise it spends nothing and names the budget that is exhausted. -func (b *knowledgeAdviserLLMBudget) take(agent string, perAgentHour, perDay int) string { +// take spends one call for agent when both budgets allow it and returns "" +// with the time the call was stamped (for refund); otherwise it spends +// nothing and names the budget that is exhausted. +func (b *knowledgeAdviserLLMBudget) take(agent string, perAgentHour, perDay int) (string, time.Time) { now := b.now() b.mu.Lock() defer b.mu.Unlock() @@ -494,14 +718,41 @@ func (b *knowledgeAdviserLLMBudget) take(agent string, perAgentHour, perDay int) switch { case perAgentHour > 0 && len(events) >= perAgentHour: b.events[agent] = events - return "agent-hour" + return "agent-hour", now case perDay > 0 && b.used >= perDay: if len(events) > 0 { b.events[agent] = events } - return "fleet-day" + return "fleet-day", now } b.events[agent] = append(events, now) b.used++ - return "" + return "", now +} + +// refund hands back the call take stamped at `at` for agent: the model never +// answered it (rate-limited, or the in-flight slot never freed), so it must +// not count against the agent's hour or the fleet's day. A stamp that has +// already rolled out of the window, or a day that has rolled over, refunds +// nothing. +func (b *knowledgeAdviserLLMBudget) refund(agent string, at time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + events := b.events[agent] + removed := false + for i := len(events) - 1; i >= 0; i-- { + if events[i].Equal(at) { + events = append(events[:i:i], events[i+1:]...) + if len(events) == 0 { + delete(b.events, agent) + } else { + b.events[agent] = events + } + removed = true + break + } + } + if removed && b.used > 0 && at.UTC().Format(time.DateOnly) == b.day { + b.used-- + } } diff --git a/src/cmd/hive/knowledge_adviser_llm_test.go b/src/cmd/hive/knowledge_adviser_llm_test.go index 9a478e9cd7..21ec2e14ce 100644 --- a/src/cmd/hive/knowledge_adviser_llm_test.go +++ b/src/cmd/hive/knowledge_adviser_llm_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "fmt" "log/slog" @@ -14,6 +15,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/internal/testutil" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/scheduler" @@ -70,6 +72,9 @@ type llmFixture struct { handler http.HandlerFunc lastReq knowledgeAdviserLLMRequest kicks map[string]scheduler.KickSummary + // waits records every wait the decorator asked for; the fixture's wait + // never sleeps. + waits []time.Duration } func (f *llmFixture) with(fn func()) { @@ -122,10 +127,7 @@ func newLLMFixture(t *testing.T, a config.AdvisorLLMConfig) *llmFixture { } return } - _ = json.NewEncoder(w).Encode(map[string]any{ - "choices": []any{map[string]any{"message": map[string]any{"content": reply}}}, - "usage": map[string]any{"prompt_tokens": 410, "completion_tokens": 17}, - }) + writeLLMReply(w, reply) })) t.Cleanup(func() { close(release); f.srv.Close() }) @@ -147,16 +149,84 @@ func newLLMFixture(t *testing.T, a config.AdvisorLLMConfig) *llmFixture { f.with(func() { k, ok = f.kicks[agent] }) return k, ok }, - http: &http.Client{}, - now: now, - logger: slog.New(slog.NewJSONHandler(f.logs, nil)), - budget: newKnowledgeAdviserLLMBudget(now), + http: &http.Client{}, + now: now, + logger: slog.New(slog.NewJSONHandler(f.logs, &slog.HandlerOptions{Level: slog.LevelDebug})), + budget: newKnowledgeAdviserLLMBudget(now), + slots: newKnowledgeAdviserLLMSlots(), + wait: func(ctx context.Context, d time.Duration) bool { + f.with(func() { f.waits = append(f.waits, d) }) + return ctx.Err() == nil + }, delivered: map[string]*knowledgeAdviserLLMKick{}, } f.kicks["developer"] = scheduler.KickSummary{ID: "developer#1", Text: "#42 Fix the tree pin [kind/bug]"} return f } +// writeLLMReply is the model server's 200 answer carrying reply. +func writeLLMReply(w http.ResponseWriter, reply string) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "choices": []any{map[string]any{"message": map[string]any{"content": reply}}}, + "usage": map[string]any{"prompt_tokens": 410, "completion_tokens": 17}, + }) +} + +const llmTest429Body = `{"error":{"message":"rate limit exceeded","code":"rate_limited"}}` + +// rateLimit makes the model answer 429 (with the Copilot-style rate-limit +// headers and, when non-empty, Retry-After) to the first n requests and the +// fixture's reply afterwards. +func (f *llmFixture) rateLimit(n int, retryAfter string) { + served := 0 + f.with(func() { + f.handler = func(w http.ResponseWriter, _ *http.Request) { + var throttle bool + var reply string + f.with(func() { + served++ + throttle = served <= n + reply = f.reply + }) + if !throttle { + writeLLMReply(w, reply) + return + } + if retryAfter != "" { + w.Header().Set("Retry-After", retryAfter) + } + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Limit", "60") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(llmTest429Body)) + } + }) +} + +func (f *llmFixture) recordedWaits() (out []time.Duration) { + f.with(func() { out = append(out, f.waits...) }) + return out +} + +// logsAt returns the decoded log lines with msg at level. +func (f *llmFixture) logsAt(t *testing.T, msg, level string) []map[string]any { + t.Helper() + var out []map[string]any + for _, line := range strings.Split(strings.TrimSpace(f.logs.String()), "\n") { + if line == "" { + continue + } + var rec map[string]any + if err := json.Unmarshal([]byte(line), &rec); err != nil { + t.Fatalf("log line %q: %v", line, err) + } + if rec["msg"] == msg && rec["level"] == level { + out = append(out, rec) + } + } + return out +} + var llmTestScope = crux.KnowledgeScope{ Kind: "path", Key: "src/pkg/tree", Path: "src/pkg/tree/pin.go", Keywords: []string{"tree"}, Action: crux.KnowledgeActionEdit, PaneTail: "⏺ Edit(src/pkg/tree/pin.go)\n token [REDACTED]", @@ -573,3 +643,247 @@ func TestKnowledgeAdviserLLM_ReasonNeverDelivered(t *testing.T) { } } } + +// 429 handling (hivecommons/hive#18151): one retry after Retry-After inside +// the timeout, the outcome recorded as rate-limited, no budget spent, and one +// advisor call in flight across the hive. + +const llmTest429Msg = "knowledge advisor: model endpoint rate limited (429)" + +func TestKnowledgeAdviserLLM_RateLimitedRetriesOnceThenRecords(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank}) + f.rateLimit(2, "1") + if got := f.d.Advise("developer", llmTestScope); len(got) != 0 { + t.Errorf("rate-limited delivered %v", slugsOf(got)) + } + if n := f.callCount(); n != 2 { + t.Errorf("calls = %d, want exactly one retry (2)", n) + } + if w := f.recordedWaits(); len(w) != 1 || w[0] != time.Second { + t.Errorf("waits = %v, want [1s] from Retry-After", w) + } + d := f.lastDecision(t) + if d["verdict"] != knowledgeVerdictRateLimited || d["outcome"] != knowledgeVerdictRateLimited { + t.Errorf("decision = %v, want verdict and outcome rate-limited", d) + } + if d["retried"] != true || d["retry_after_s"] != float64(1) { + t.Errorf("decision retried/retry_after_s = %v/%v", d["retried"], d["retry_after_s"]) + } + warns := f.logsAt(t, llmTest429Msg, "WARN") + if len(warns) != 1 { + t.Fatalf("WARN lines = %d, want 1 (first 429 in the window); logs:\n%s", len(warns), f.logs.String()) + } + w := warns[0] + if w["header_retry_after"] != "1" || w["header_x_ratelimit_remaining"] != "0" || w["header_x_ratelimit_limit"] != "60" { + t.Errorf("WARN line lacks the rate-limit headers: %v", w) + } + if body, _ := w["body"].(string); !strings.Contains(body, "rate limit exceeded") { + t.Errorf("WARN body = %q", body) + } + if debugs := f.logsAt(t, llmTest429Msg, "DEBUG"); len(debugs) != 1 || debugs[0]["retried"] != true { + t.Errorf("DEBUG lines = %v, want the retry's 429 only", debugs) + } +} + +func TestKnowledgeAdviserLLM_RateLimitedRetrySucceeds(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank}) + f.setReply(`{"slug": "first", "reason": "editing the pin"}`) + f.rateLimit(1, "0.5") + if got := slugsOf(f.d.Advise("developer", llmTestScope)); strings.Join(got, ",") != "first" { + t.Errorf("advice after the retry = %v, want [first]", got) + } + if n := f.callCount(); n != 2 { + t.Errorf("calls = %d, want 2", n) + } + if d := f.lastDecision(t); d["outcome"] != "delivered" || d["verdict"] != "pick" { + t.Errorf("decision = %v", d) + } +} + +func TestKnowledgeAdviserLLM_RateLimitedWithoutRetryAfterBacksOffWithJitter(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank}) + f.rateLimit(2, "") + f.d.Advise("developer", llmTestScope) + w := f.recordedWaits() + if len(w) != 1 || w[0] < knowledgeAdviserLLMRetryBackoff || w[0] >= 2*knowledgeAdviserLLMRetryBackoff { + t.Errorf("waits = %v, want one in [%s, %s)", w, knowledgeAdviserLLMRetryBackoff, 2*knowledgeAdviserLLMRetryBackoff) + } + if d := f.lastDecision(t); d["outcome"] != knowledgeVerdictRateLimited || d["retry_after_s"] != float64(0) { + t.Errorf("decision = %v", d) + } +} + +func TestKnowledgeAdviserLLM_RateLimitedSkipsRetryBeyondTimeout(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank, TimeoutS: 5}) + f.rateLimit(2, "30") + f.d.Advise("developer", llmTestScope) + if n := f.callCount(); n != 1 { + t.Errorf("calls = %d, want 1: a 30s Retry-After cannot fit a 5s timeout", n) + } + if w := f.recordedWaits(); len(w) != 0 { + t.Errorf("waited %v for a retry that cannot fit", w) + } + if d := f.lastDecision(t); d["outcome"] != knowledgeVerdictRateLimited || d["retried"] != false || d["retry_after_s"] != float64(30) { + t.Errorf("decision = %v", d) + } +} + +func TestKnowledgeAdviserLLM_RateLimitedOnErrorMatcherFallsBack(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank, OnError: config.AdvisorLLMOnErrorMatcher}) + f.rateLimit(2, "1") + if got := slugsOf(f.d.Advise("developer", llmTestScope)); strings.Join(got, ",") != "first" { + t.Errorf("on_error matcher = %v, want [first]", got) + } + if d := f.lastDecision(t); d["outcome"] != "fallback-matcher" || d["verdict"] != knowledgeVerdictRateLimited { + t.Errorf("decision = %v", d) + } +} + +func TestKnowledgeAdviserLLM_RateLimitedDoesNotConsumeBudget(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank, MaxCallsPerAgentHour: 1}) + f.rateLimit(2, "1") + f.d.Advise("developer", llmTestScope) + if d := f.lastDecision(t); d["outcome"] != knowledgeVerdictRateLimited { + t.Fatalf("decision = %v", d) + } + f.d.budget.mu.Lock() + events, used := len(f.d.budget.events["developer"]), f.d.budget.used + f.d.budget.mu.Unlock() + if events != 0 || used != 0 { + t.Errorf("after a rate-limited decision agent events = %d fleet used = %d, want 0/0", events, used) + } + // The one slot of the hour is still free: the next decision reaches the + // model instead of being over-budget. + f.with(func() { f.handler = nil }) + f.d.Advise("developer", llmTestScope) + if n := f.callCount(); n != 3 { + t.Errorf("calls = %d, want 3 (the rate-limited pair plus one real call)", n) + } + if d := f.lastDecision(t); d["outcome"] != "null" { + t.Errorf("decision after the throttled one = %v, want a real null verdict, not over-budget", d) + } +} + +func TestKnowledgeAdviserLLM_GlobalCapSerializesAgents(t *testing.T) { + f := newLLMFixture(t, config.AdvisorLLMConfig{Mode: config.AdvisorLLMModeRank}) + f.setKick("scanner", scheduler.KickSummary{ID: "scanner#1", Text: "#7 Scan"}) + release := make(chan struct{}) + var seq []string + served := 0 + f.with(func() { + f.handler = func(w http.ResponseWriter, _ *http.Request) { + var n int + f.with(func() { + served++ + n = served + seq = append(seq, fmt.Sprintf("start%d", n)) + }) + if n == 1 { + <-release + } + f.with(func() { seq = append(seq, fmt.Sprintf("end%d", n)) }) + writeLLMReply(w, `{"slug": null, "reason": "reading"}`) + } + }) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + f.d.Advise("developer", llmTestScope) + }() + testutil.Eventually(t, 5*time.Second, func() bool { return f.callCount() == 1 }, "first agent's call did not reach the model") + wg.Add(1) + go func() { + defer wg.Done() + f.d.Advise("scanner", llmTestScope) + }() + testutil.Eventually(t, 5*time.Second, func() bool { return f.d.waiting.Load() == 1 }, "second agent is not waiting on the hive-wide slot") + if n := f.callCount(); n != 1 { + t.Fatalf("calls = %d while the first is in flight, want 1", n) + } + close(release) + wg.Wait() + if n := f.callCount(); n != 2 { + t.Errorf("calls = %d, want 2", n) + } + var got string + f.with(func() { got = strings.Join(seq, ",") }) + if got != "start1,end1,start2,end2" { + t.Errorf("call sequence = %s, want start1,end1,start2,end2", got) + } + if n := len(f.decisions(t)); n != 2 { + t.Errorf("decisions logged = %d, want 2", n) + } + if f.d.waiting.Load() != 0 { + t.Errorf("waiting = %d after both calls returned", f.d.waiting.Load()) + } +} + +func TestKnowledgeAdviserLLMBudgetRefund(t *testing.T) { + clock := time.Date(2026, 11, 13, 20, 0, 0, 0, time.UTC) + b := newKnowledgeAdviserLLMBudget(func() time.Time { return clock }) + if over, _ := b.take("developer", 2, 10); over != "" { + t.Fatalf("first take over = %q", over) + } + clock = clock.Add(time.Minute) + over, stamp := b.take("developer", 2, 10) + if over != "" { + t.Fatalf("second take over = %q", over) + } + if over, _ := b.take("developer", 2, 10); over != "agent-hour" { + t.Fatalf("third take over = %q, want agent-hour", over) + } + b.refund("developer", stamp) + b.mu.Lock() + events, used := len(b.events["developer"]), b.used + b.mu.Unlock() + if events != 1 || used != 1 { + t.Errorf("after refund events = %d used = %d, want 1/1", events, used) + } + if over, _ := b.take("developer", 2, 10); over != "" { + t.Errorf("take after refund over = %q, want the slot back", over) + } + // A stamp that is not there refunds no event and never drives used negative. + b.refund("developer", stamp.Add(time.Hour)) + b.refund("other", stamp) + b.mu.Lock() + events, used = len(b.events["developer"]), b.used + b.mu.Unlock() + if events != 2 || used != 2 { + t.Errorf("after bogus refunds events = %d used = %d, want 2/2 (the take after the refund spent one again)", events, used) + } +} + +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, 11, 13, 20, 0, 0, 0, time.UTC) + for in, want := range map[string]time.Duration{ + "": 0, + "3": 3 * time.Second, + " 1.5 ": 1500 * time.Millisecond, + "0": 0, + "-2": 0, + "soon": 0, + now.Add(7 * time.Second).UTC().Format(http.TimeFormat): 7 * time.Second, + now.Add(-7 * time.Second).UTC().Format(http.TimeFormat): 0, + } { + if got := parseRetryAfter(in, now); got != want { + t.Errorf("parseRetryAfter(%q) = %s, want %s", in, got, want) + } + } +} + +func TestKnowledgeAdviserLLMRetryWait(t *testing.T) { + now := time.Date(2026, 11, 13, 20, 0, 0, 0, time.UTC) + ctx, cancel := context.WithDeadline(context.Background(), now.Add(time.Duration(config.DefaultAdvisorLLMTimeoutS)*time.Second)) + defer cancel() + if wait, ok := knowledgeAdviserLLMRetryWait(ctx, time.Second, now); !ok || wait != time.Second { + t.Errorf("1s Retry-After inside 15s = %s/%v, want 1s/true", wait, ok) + } + tooLong := time.Duration(config.DefaultAdvisorLLMTimeoutS)*time.Second - knowledgeAdviserLLMRetryHeadroom + time.Second + if _, ok := knowledgeAdviserLLMRetryWait(ctx, tooLong, now); ok { + t.Errorf("%s Retry-After leaves no headroom inside 15s, want no retry", tooLong) + } + if wait, ok := knowledgeAdviserLLMRetryWait(context.Background(), 0, now); !ok || wait < knowledgeAdviserLLMRetryBackoff || wait >= 2*knowledgeAdviserLLMRetryBackoff { + t.Errorf("no Retry-After, no deadline = %s/%v, want jittered backoff/true", wait, ok) + } +} diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md index 9cbac7e37b..e3527ebf04 100644 --- a/src/docs/agent-knowledge-and-memory.md +++ b/src/docs/agent-knowledge-and-memory.md @@ -152,7 +152,8 @@ flowchart LR G -->|rank| R[model: one candidate slug or null] R -->|slug in candidates| D2[deliver that entry] R -->|null / invalid| X2[nothing] - R -->|timeout / error / over budget| E{on_error} + R -->|429| W[wait Retry-After, retry once] --> R + R -->|timeout / error / rate-limited / over budget| E{on_error} E -->|silence| X2 E -->|matcher| D3[phase-1 top-1] ``` @@ -174,18 +175,37 @@ flowchart LR `hive-knowledge-advisor:<agent>` so proxy spend logs attribute cost per agent. The answer must be `{"slug": <candidate>|null, "reason": "..."}`; a slug that is not exactly a candidate, or a malformed answer, is `invalid` - and treated as `null`. There is no retry, and the `reason` is logged, never - typed. + and treated as `null`. A wrong answer is never retried (only a 429 is, once; + see Pacing), and the `reason` is logged, never typed. - **Modes.** `shadow` delivers phase 1's top entry and records what the model would have picked. `rank` delivers the model's pick; with `holdout_percent`, the kick identity is hashed into a delivery or holdout arm, and holdout kicks make the call and log the pick but type nothing. A valid `null` never falls back to the matcher; `on_error: matcher` only - covers timeouts, endpoint errors and exhausted budgets. + covers timeouts, endpoint errors, rate-limited calls and exhausted budgets. - **Budgets.** A rolling one-hour window per agent (`max_calls_per_agent_hour`, default and ceiling 6) and a per-UTC-day fleet counter (`max_calls_per_day`, default and ceiling 300). An exhausted budget - makes no request and is handled like an error. + makes no request and is handled like an error. A call the endpoint answered + with 429 is handed back to both budgets: a throttled hour must not also + become an over-budget hour. +- **Pacing (429s).** The advisor usually shares its upstream quota with the + agents (on the Copilot API, the same user token every `copilot` agent + spends), and holdout kicks make the call too, so N agents entering scopes + in the same second used to fan out N calls against one quota and lose + (hivecommons/hive#18151). At most one advisor call is now in flight across + the whole hive, behind the runtime's one-per-agent guard; a decision that + cannot get the slot inside `timeout_s` records `timeout` without a request + and without spending budget. A 429 is retried exactly once after the + upstream's `Retry-After` (seconds or HTTP date), or after a 2 to 4 s + jittered backoff when it sent none, and only when that wait leaves at least + 3 s of `timeout_s` for the retry; otherwise the decision is recorded + `rate-limited` straight away. A decision that is still throttled after the + retry is `rate-limited`, distinct from `error`. The first 429 in each + five-minute window logs `knowledge advisor: model endpoint rate limited + (429)` at WARN with `retry_after_s`, the `header_retry_after` and + `header_x_ratelimit_*` values and the response body; the rest of the window + logs at DEBUG. - **Decision log.** Every model decision writes one line: ``` @@ -195,17 +215,20 @@ flowchart LR ``` `verdict` is what the model produced (`pick`, `null`, `invalid`, - `timeout`, `over-budget`, `error`); `outcome` is what happened to delivery - (`delivered`, `held-out`, `gated` when the kick moved during the call, - `fallback-matcher`, or the verdict when nothing was typed). Over-budget - lines also carry `budget=agent-hour|fleet-day`. Delivered lines still land - in the prompt history under trigger `knowledge-advisor`. + `timeout`, `over-budget`, `rate-limited`, `error`); `outcome` is what + happened to delivery (`delivered`, `held-out`, `gated` when the kick moved + during the call, `fallback-matcher`, or the verdict when nothing was typed). + Over-budget lines also carry `budget=agent-hour|fleet-day`; rate-limited + lines carry `retry_after_s` and `retried`. Delivered lines still land in + the prompt history under trigger `knowledge-advisor`. - **Decision store.** Each decision, held-out ones included, also appends one compact record (agent, kick id, the kick's primed issue/PR refs, mode, arm `shadow|deliver|holdout`, outcome, slug; never the reason or pane text) to `/data/knowledge-advisor-decisions.jsonl`, which the retro lane reads to assign each bead its advisor arm (see - [retro-lane.md](retro-lane.md#knowledge-advisor-arm)). + [retro-lane.md](retro-lane.md#knowledge-advisor-arm)). The record schema is + unchanged by `rate-limited`: it is one more `outcome` value, which the lane + counts but excludes from the deliver arm. **Rollout.** Start with `mode: shadow` for about a week: delivery is unchanged, and the decision lines show how often the model says `null`, how diff --git a/src/docs/retro-lane.md b/src/docs/retro-lane.md index 7d612a5ce4..efc4b87a23 100644 --- a/src/docs/retro-lane.md +++ b/src/docs/retro-lane.md @@ -27,7 +27,9 @@ built from the advisor's decision records and joined by the issue ref: - `AdvisorArm`: `none` (no decisions), `shadow` (shadow mode, delivery unchanged), `deliver` (rank mode, advice delivered), `holdout` (every kick was held out: the model ranked but nothing was typed), or `mixed` (kicks in more than one arm). -- `AdvisorDecisions`, `AdvisorDelivered`, `AdvisorHeldOut`: counters over those decisions. +- `AdvisorDecisions`, `AdvisorDelivered`, `AdvisorHeldOut`, `AdvisorRateLimited`: counters over + those decisions. A `rate-limited` decision (the model endpoint answered 429) is counted but + assigns no arm, so a throttled hour does not move a bead into `deliver`. Only these counters reach the analysis model; the advisor's reasons and pane text never do. diff --git a/src/pkg/retro/analysis.go b/src/pkg/retro/analysis.go index c0a5860d22..790456557e 100644 --- a/src/pkg/retro/analysis.go +++ b/src/pkg/retro/analysis.go @@ -138,9 +138,9 @@ func buildAnalysisPrompt(record RetroRecord, findings []Finding) []chatMessage { "Return ONLY one JSON object with keys root_cause_hypothesis, process_improvement, generalizable, generalizable_lesson. " + "Use concise, non-secret, process-level language. Do not invent facts. If the lesson is too specific to this bead, set generalizable=false and generalizable_lesson=\"\"." var b strings.Builder - fmt.Fprintf(&b, "RETRO_RECORD\nbead_id: %s\ntitle: %s\ntype: %s\nactor: %s\nissue: %s\npr: %s\npr_state: %s\nkicks_received: %d\nci_failures: %d\nfix_attempts: %d\nadvisor_arm: %s\nadvisor_decisions: %d\nadvisor_delivered: %d\nadvisor_held_out: %d\ndrift_pauses: %d\nplan_revisions_before_approval: %d\npr_rework_commits_after_review: %d\nrollback_events: %d\nscope_user: %s\nscope_repo: %s\nscope_change_class: %s\nautonomy_level: %s\nclaim_to_close: %s\n\n", + fmt.Fprintf(&b, "RETRO_RECORD\nbead_id: %s\ntitle: %s\ntype: %s\nactor: %s\nissue: %s\npr: %s\npr_state: %s\nkicks_received: %d\nci_failures: %d\nfix_attempts: %d\nadvisor_arm: %s\nadvisor_decisions: %d\nadvisor_delivered: %d\nadvisor_held_out: %d\nadvisor_rate_limited: %d\ndrift_pauses: %d\nplan_revisions_before_approval: %d\npr_rework_commits_after_review: %d\nrollback_events: %d\nscope_user: %s\nscope_repo: %s\nscope_change_class: %s\nautonomy_level: %s\nclaim_to_close: %s\n\n", record.BeadID, truncate(record.Title, maxRecordTitleChars), record.Type, record.Actor, record.IssueRef, record.PRRef, record.PRState, - record.KicksReceived, record.CIFailureCount, record.FixAttempts, firstNonEmpty(record.AdvisorArm, AdvisorArmNone), record.AdvisorDecisions, record.AdvisorDelivered, record.AdvisorHeldOut, record.DriftPauses, record.PlanRevisionsBeforeApproval, record.PRReworkCommitsAfterReview, + record.KicksReceived, record.CIFailureCount, record.FixAttempts, firstNonEmpty(record.AdvisorArm, AdvisorArmNone), record.AdvisorDecisions, record.AdvisorDelivered, record.AdvisorHeldOut, record.AdvisorRateLimited, record.DriftPauses, record.PlanRevisionsBeforeApproval, record.PRReworkCommitsAfterReview, record.RollbackEvents, record.ScopeUser, record.ScopeRepo, record.ScopeChangeClass, record.AutonomyLevel, record.ClaimToClose.String()) b.WriteString("DETERMINISTIC_FINDINGS\n") for _, f := range findings { diff --git a/src/pkg/retro/retro.go b/src/pkg/retro/retro.go index 805b7591e2..0bb50d215d 100644 --- a/src/pkg/retro/retro.go +++ b/src/pkg/retro/retro.go @@ -92,6 +92,7 @@ type RetroRecord struct { AdvisorDecisions int AdvisorDelivered int AdvisorHeldOut int + AdvisorRateLimited int DriftPauses int PlanRevisionsBeforeApproval int PRReworkCommitsAfterReview int @@ -540,10 +541,17 @@ func Reconstruct(b *beads.Bead, tl *timeline.Store, attempts AttemptReader) Retr return r } +// AdvisorOutcomeRateLimited is the decision outcome recorded when the model +// endpoint answered 429 (hivecommons/hive#18151). The agent received nothing +// and no budget was spent, so it is counted but assigns no arm: it is neither +// a delivery nor a holdout. +const AdvisorOutcomeRateLimited = "rate-limited" + // ApplyAdvisorDecisions fills the advisor arm counters from the decisions // recorded for the record's issue. Holdout kicks are held-out, shadow-mode -// kicks are the shadow arm, every other rank-mode decision is the deliver arm; -// kicks in more than one arm make the bead mixed. +// kicks are the shadow arm, every other rank-mode decision is the deliver arm +// except rate-limited ones, which are counted and excluded; kicks in more than +// one arm make the bead mixed. func ApplyAdvisorDecisions(r *RetroRecord, reader AdvisorDecisionReader) { r.AdvisorArm = AdvisorArmNone if reader == nil || r.IssueRef == "" { @@ -553,6 +561,8 @@ func ApplyAdvisorDecisions(r *RetroRecord, reader AdvisorDecisionReader) { for _, d := range reader.AdvisorDecisions(r.IssueRef) { r.AdvisorDecisions++ switch d.Outcome { + case AdvisorOutcomeRateLimited: + r.AdvisorRateLimited++ case "held-out": r.AdvisorHeldOut++ arms[AdvisorArmHoldout] = true diff --git a/src/pkg/retro/retro_test.go b/src/pkg/retro/retro_test.go index 188b680130..fa42110a34 100644 --- a/src/pkg/retro/retro_test.go +++ b/src/pkg/retro/retro_test.go @@ -389,23 +389,29 @@ func TestApplyAdvisorDecisionsArms(t *testing.T) { "o/r#1": {{Kick: "dev#1", Mode: "rank", Outcome: "held-out"}, {Kick: "dev#2", Mode: "rank", Outcome: "held-out"}}, "o/r#2": {{Kick: "dev#3", Mode: "rank", Outcome: "held-out"}, {Kick: "dev#4", Mode: "rank", Outcome: "delivered"}}, "o/r#4": {{Kick: "dev#5", Mode: "shadow", Outcome: "delivered"}}, + // A rate-limited decision is counted but assigns no arm (#18151): the + // bead stays in the arm its other decisions put it in, or none. + "o/r#5": {{Kick: "dev#6", Mode: "rank", Outcome: AdvisorOutcomeRateLimited}}, + "o/r#6": {{Kick: "dev#7", Mode: "rank", Outcome: AdvisorOutcomeRateLimited}, {Kick: "dev#8", Mode: "rank", Outcome: "held-out"}}, } cases := []struct { ref string arm string decisions, delivered int - heldOut int + heldOut, rateLimited int }{ - {"o/r#1", "holdout", 2, 0, 2}, - {"o/r#2", "mixed", 2, 1, 1}, - {"o/r#3", "none", 0, 0, 0}, - {"o/r#4", "shadow", 1, 1, 0}, + {"o/r#1", "holdout", 2, 0, 2, 0}, + {"o/r#2", "mixed", 2, 1, 1, 0}, + {"o/r#3", "none", 0, 0, 0, 0}, + {"o/r#4", "shadow", 1, 1, 0, 0}, + {"o/r#5", "none", 1, 0, 0, 1}, + {"o/r#6", "holdout", 2, 0, 1, 1}, } for _, tt := range cases { r := RetroRecord{IssueRef: tt.ref} ApplyAdvisorDecisions(&r, reader) - if r.AdvisorArm != tt.arm || r.AdvisorDecisions != tt.decisions || r.AdvisorDelivered != tt.delivered || r.AdvisorHeldOut != tt.heldOut { - t.Fatalf("%s = %s/%d/%d/%d, want %s/%d/%d/%d", tt.ref, r.AdvisorArm, r.AdvisorDecisions, r.AdvisorDelivered, r.AdvisorHeldOut, tt.arm, tt.decisions, tt.delivered, tt.heldOut) + if r.AdvisorArm != tt.arm || r.AdvisorDecisions != tt.decisions || r.AdvisorDelivered != tt.delivered || r.AdvisorHeldOut != tt.heldOut || r.AdvisorRateLimited != tt.rateLimited { + t.Fatalf("%s = %s/%d/%d/%d/%d, want %s/%d/%d/%d/%d", tt.ref, r.AdvisorArm, r.AdvisorDecisions, r.AdvisorDelivered, r.AdvisorHeldOut, r.AdvisorRateLimited, tt.arm, tt.decisions, tt.delivered, tt.heldOut, tt.rateLimited) } } r := RetroRecord{IssueRef: "o/r#1"} From 4d140e6b3f35c6fcb5fcece2bfa0f91c295f5d0f Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 00:25:37 +0000 Subject: [PATCH 192/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.249.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.249.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 13 +++++++++++++ changelog.d/added-reference-repo-tier-accessor.md | 1 - changelog.d/added-sessions-backend.md | 1 - changelog.d/added-sessions-ui.md | 1 - .../fixed-18128-native-queue-policy-reason.md | 1 - changelog.d/fixed-18137-stray-replica-overlay.md | 1 - 6 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-reference-repo-tier-accessor.md delete mode 100644 changelog.d/added-sessions-backend.md delete mode 100644 changelog.d/added-sessions-ui.md delete mode 100644 changelog.d/fixed-18128-native-queue-policy-reason.md delete mode 100644 changelog.d/fixed-18137-stray-replica-overlay.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2803720f9f..998ab5329a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.249.0) + +### Added + +- `config.ProjectConfig` gains a `ReferenceRepoTier(slug)` accessor (and `ReferenceRepo.Tier()`) returning a reference repo's write tier, so the proxy, kick renderer and dashboard share one source of truth (#18018). +- Dashboard Sessions backend (off by default, `dashboard.sessions.enabled` / `HIVE_SESSIONS_ENABLED`): named, shared, interactive CLI terminals behind `/api/sessions` (create, rename, close, resume, scrollback, input and upload), with per-session HOME and a credential-free environment, no permission-bypass flags, per-spoke and per-user limits, an idle sweeper and `session_*` audit entries; `/api/terminal/handoff` accepts `{"session":"<id>"}`. +- Dashboard: a new Sessions section (when sessions are enabled) with tabbed, named, grouped, shared CLI terminals, a send/attach input bar and a read-only scrollback view for the read role; it replaces the Inception wizard UI and keeps Campaigns and Jam on a tab. + +### Fixed + +- `native-queue-policy` now names the condition that made a PR ineligible for admission (`is a draft`, `is held by label needs-human`, `head moved to <sha>`, `base moved to v6`, `is closed`) instead of the single opaque `head/base moved, draft, closed, or held` line, and says it is PR state rather than a CI failure. Several held or draft PRs failing with the same generic text looked like a broken shared workflow and were filed as a shared-CI incident (#18128); the check's fail-closed verdict was correct on every one of them. +- **A per-agent overlay for a derived replica no longer crash-loops the hive** (#18137). Dashboard saves on a replica such as `reviewer-2` used to write `reviewer-2.yaml` into `data.agents_dir` as if it were a base agent, and the next restart failed with `replicas:2 would create reviewer-2, but an agent with that name already exists` before the dashboard could bind. Replica edits are now written to the base agent's overlay, and a stray `<base>-N.yaml` already on disk is skipped at boot with an ERROR log naming the file, so the hive starts and the replica is re-derived from its base. + ## 2026-11-13 (v5.248.1) ### Fixed diff --git a/changelog.d/added-reference-repo-tier-accessor.md b/changelog.d/added-reference-repo-tier-accessor.md deleted file mode 100644 index 5b7891cedd..0000000000 --- a/changelog.d/added-reference-repo-tier-accessor.md +++ /dev/null @@ -1 +0,0 @@ -- `config.ProjectConfig` gains a `ReferenceRepoTier(slug)` accessor (and `ReferenceRepo.Tier()`) returning a reference repo's write tier, so the proxy, kick renderer and dashboard share one source of truth (#18018). diff --git a/changelog.d/added-sessions-backend.md b/changelog.d/added-sessions-backend.md deleted file mode 100644 index fc9173aaab..0000000000 --- a/changelog.d/added-sessions-backend.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard Sessions backend (off by default, `dashboard.sessions.enabled` / `HIVE_SESSIONS_ENABLED`): named, shared, interactive CLI terminals behind `/api/sessions` (create, rename, close, resume, scrollback, input and upload), with per-session HOME and a credential-free environment, no permission-bypass flags, per-spoke and per-user limits, an idle sweeper and `session_*` audit entries; `/api/terminal/handoff` accepts `{"session":"<id>"}`. diff --git a/changelog.d/added-sessions-ui.md b/changelog.d/added-sessions-ui.md deleted file mode 100644 index 3586fd8378..0000000000 --- a/changelog.d/added-sessions-ui.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: a new Sessions section (when sessions are enabled) with tabbed, named, grouped, shared CLI terminals, a send/attach input bar and a read-only scrollback view for the read role; it replaces the Inception wizard UI and keeps Campaigns and Jam on a tab. diff --git a/changelog.d/fixed-18128-native-queue-policy-reason.md b/changelog.d/fixed-18128-native-queue-policy-reason.md deleted file mode 100644 index 1773921570..0000000000 --- a/changelog.d/fixed-18128-native-queue-policy-reason.md +++ /dev/null @@ -1 +0,0 @@ -- `native-queue-policy` now names the condition that made a PR ineligible for admission (`is a draft`, `is held by label needs-human`, `head moved to <sha>`, `base moved to v6`, `is closed`) instead of the single opaque `head/base moved, draft, closed, or held` line, and says it is PR state rather than a CI failure. Several held or draft PRs failing with the same generic text looked like a broken shared workflow and were filed as a shared-CI incident (#18128); the check's fail-closed verdict was correct on every one of them. diff --git a/changelog.d/fixed-18137-stray-replica-overlay.md b/changelog.d/fixed-18137-stray-replica-overlay.md deleted file mode 100644 index 09f4900827..0000000000 --- a/changelog.d/fixed-18137-stray-replica-overlay.md +++ /dev/null @@ -1 +0,0 @@ -- **A per-agent overlay for a derived replica no longer crash-loops the hive** (#18137). Dashboard saves on a replica such as `reviewer-2` used to write `reviewer-2.yaml` into `data.agents_dir` as if it were a base agent, and the next restart failed with `replicas:2 would create reviewer-2, but an agent with that name already exists` before the dashboard could bind. Replica edits are now written to the base agent's overlay, and a stray `<base>-N.yaml` already on disk is skipped at boot with an ERROR log naming the file, so the hive starts and the replica is re-derived from its base. From ba875a34932d010691fd619ec7b492406ec4469d Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 19:37:24 -0500 Subject: [PATCH 193/223] =?UTF-8?q?=E2=9C=A8=20hub:=20self-service=20(byo-?= =?UTF-8?q?)=20hive=20registration=20API=20with=20GitHub=20admin=20ownersh?= =?UTF-8?q?ip=20proof,=20rotate=20and=20revoke=20(#18155)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Behind HIVE_HUB_SELFSERVICE_REGISTRATION (default off), a signed-in user can register a hive they run themselves. The hub builds the ID as byo-<github login>-<name>, refuses reserved and duplicate IDs, proves permissions.admin on every owner/name repo with the caller's own stored GitHub token (refusing the whole registration by repo name otherwise), enforces HIVE_HUB_SELFSERVICE_MAX_PER_USER (default 5), and returns the raw heartbeat key once with Docker Compose, Podman and Kubernetes snippets. The owner can list, rotate and revoke; a hub admin can revoke. Another user's hive and the whole family while the flag is off answer 404. Every decision writes an audit: line that never carries key material. Part of #18060 Fixes #18144 Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...dded-18144-selfservice-registration-api.md | 1 + src/docs/api-reference/hub-server.md | 63 +- src/docs/audit-log.md | 18 + src/docs/env-vars.md | 3 + src/pkg/hub/selfservice_api.go | 587 +++++++++++++++++ src/pkg/hub/selfservice_api_test.go | 618 ++++++++++++++++++ src/pkg/hub/server.go | 18 +- 7 files changed, 1276 insertions(+), 32 deletions(-) create mode 100644 changelog.d/added-18144-selfservice-registration-api.md create mode 100644 src/pkg/hub/selfservice_api.go create mode 100644 src/pkg/hub/selfservice_api_test.go diff --git a/changelog.d/added-18144-selfservice-registration-api.md b/changelog.d/added-18144-selfservice-registration-api.md new file mode 100644 index 0000000000..d0a85fe274 --- /dev/null +++ b/changelog.d/added-18144-selfservice-registration-api.md @@ -0,0 +1 @@ +- Hub: signed-in users can register a hive they run themselves ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18144](https://github.com/hivecommons/hive/issues/18144)). Behind `HIVE_HUB_SELFSERVICE_REGISTRATION` (default off), `POST /api/hub/selfservice/hives` builds the ID as `byo-<login>-<name>`, proves `permissions.admin` on every listed repo with the caller's own GitHub token (refusing the whole registration by repo name otherwise), enforces `HIVE_HUB_SELFSERVICE_MAX_PER_USER` (default 5), and returns the heartbeat key once with Docker Compose, Podman and Kubernetes snippets; the owner can list, rotate and revoke their entries, a hub admin can revoke any, and every decision writes an `audit:` line without key material. The dashboard form follows in a later PR. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index c13fed2c42..8666408551 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,47 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks/<hash>/<YYYY-MM>.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1807` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1808` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1819` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1820` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1739` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1740` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1741` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1742` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1743` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1754` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1755` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1756` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1757` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1743` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1744` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1745` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1746` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1747` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1758` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1759` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1760` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1761` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=<id>[,<id>…]` (repeatable) and `?level=<n>` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence-<hive|fleet>-<from>-<to>.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#<n>`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1765` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1775` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1776` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1777` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1778` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1779` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1780` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1788` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1789` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1796` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1798` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1815` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1820` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1822` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1769` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1779` | +| `GET` | `/api/hub/selfservice/status` | Hub auth | Self-service registration status: `{enabled, max_per_user, count, expiry}`; always answers so the dashboard can decide whether **Register your own hive** opens the form or the self-hosting guide ([#18060](https://github.com/hivecommons/hive/issues/18060)) | `pkg/hub/server.go:1783` | +| `GET` | `/api/hub/selfservice/hives` | Hub auth | The caller's own self-service (`byo-`) hives: `{hives[]}` of `{id, owner, org, repos, dashboard_url, created_at, ownership_checked_at, last_accepted_heartbeat_at, online}`; never a key or a hash. `404` while `HIVE_HUB_SELFSERVICE_REGISTRATION` is off | `pkg/hub/server.go:1784` | +| `POST` | `/api/hub/selfservice/hives` | Hub auth | Register a self-hosted hive: body `{name, org, repos[], dashboard_url}`; the ID is `byo-<github login>-<name>`. Proves `permissions.admin` on every `owner/name` repo with the caller's own stored GitHub token (`403` naming the refused repo, nothing stored); `400` invalid or reserved name, bare-org repo, or non-`https`/private `dashboard_url`; `409` ID already registered; `429` over `HIVE_HUB_SELFSERVICE_MAX_PER_USER`. `201 {hive, key, hub_url, snippets{docker_compose, podman, kubernetes}}`: the raw key appears here once and only its hash is stored. `404` while the flag is off | `pkg/hub/server.go:1785` | +| `POST` | `/api/hub/selfservice/hives/{id}/rotate` | Hub auth | Owner only: mint a new heartbeat key for `{id}`; the previous key keeps authenticating for `overlap` (10 minutes). `200 {id, key, hub_url, overlap, snippets}`, raw key once. Another user's hive, a missing hive and a hub admin who does not own it all get `404` | `pkg/hub/server.go:1786` | +| `DELETE` | `/api/hub/selfservice/hives/{id}` | Hub auth | Owner or hub admin: revoke the key and remove the registry entry. `200 {removed, id}`; another user's hive is `404` | `pkg/hub/server.go:1787` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1788` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1789` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1790` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1791` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1792` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1793` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1794` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1800` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1801` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1808` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1809` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1810` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1827` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1832` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1834` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/docs/audit-log.md b/src/docs/audit-log.md index f2ae973405..202a8423d2 100644 --- a/src/docs/audit-log.md +++ b/src/docs/audit-log.md @@ -190,6 +190,24 @@ Things to know when you query them: older per-task files are pruned when the next attestation is stored, at most once an hour. +## Hub self-service registration lines + +The hub does not write `audit.jsonl`; its accountability record is the +`audit: <action>` lines in the hub server log (the same place `access +granted`, `hub admin granted` and `registry entry removed` land). The +self-service registration API ([#18060](https://github.com/hivecommons/hive/issues/18060), +`pkg/hub/selfservice_api.go`) writes one line per decision. `user` is the hub +login of the caller; `hive` is the `byo-` ID the request named or would have +produced (empty when the name never became an ID). No line ever carries a key, +a key hash, the master secret or the user's GitHub token. + +| Action | Written when | Keys | +| --- | --- | --- | +| `selfservice_hive_registered` | A registration passed every check and the key was minted | `user`, `hive`, `repos` (count), `org` | +| `selfservice_hive_rotated` | The owner rotated the heartbeat key | `user`, `hive`, `overlap` (how long the previous key keeps authenticating) | +| `selfservice_hive_revoked` | The owner or a hub admin revoked the hive; the registry entry is removed in the same request and `registry entry removed` follows | `user`, `hive`, `owner`, `by_admin` | +| `selfservice_hive_refused` | A registration was turned away and nothing was stored | `user`, `hive`, `reason` (`invalid_name`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `per_user_cap` with `max_per_user`, `no_github_token`, `no_github_login`, `not_repo_admin` with `repo`, `ownership_check_error` with `repo`) | + ## Rotation and retention Rotation is handled by lumberjack: diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index dbc283da19..4b4be0815d 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -84,6 +84,9 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_HUB_KIOSK_CTA_TEXT` | No | `🐝 Want this for your repos?` (`kioskcta.DefaultText`) | Copy line above the URL on that badge. | | `HIVE_HUB_LEGACY_COOKIE_DOMAIN` | No | none | Optional transition-only hub SSO cookie domain to expire alongside the active cookie domain while accepting any carried `hive_hub_user` cookie value during a host migration. Writes still target the domain derived from `HIVE_HUB_PUBLIC_URL`; unset disables the extra legacy-domain cleanup. | | `HIVE_HUB_SECRET` | Required for spokes registered to a protected hub; optional for a standalone hub with `/data/saas/hub-secret.key` | `/data/saas/hub-secret.key` on the hub when present; no fallback for spoke heartbeat auth | Bearer secret for spoke heartbeats and hub/spoke SaaS APIs. | +| `HIVE_HUB_SELFSERVICE_REGISTRATION` | No | off | Enables the self-service registration API (`/api/hub/selfservice/*`) that lets a signed-in user register a hive they run themselves under the reserved `byo-<login>-<name>` ID namespace, after proving `permissions.admin` on every listed repo with their own GitHub token ([#18060](https://github.com/hivecommons/hive/issues/18060)). Accepts `1/true/yes/on`. When off, every route but `GET /api/hub/selfservice/status` answers `404` and the status reports `enabled: false`. Read per request; no restart needed. | +| `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | No | `5` | Maximum self-service hives one user may have registered at once; the next registration answers `429` until one is removed. | +| `HIVE_HUB_SELFSERVICE_EXPIRY` | No | `24h` | How long a self-service hive may go without an accepted heartbeat before the expiry sweep removes it. Reported by `GET /api/hub/selfservice/status` as `expiry`; the sweep itself lands with the follow-up to [#18060](https://github.com/hivecommons/hive/issues/18060). Go duration syntax. | | `HIVE_HUB_AGENT_RESTART_PROBLEM_THRESHOLD` | No | `5` | Hosted hub threshold for surfacing agent restart storms on `/fleet`. When an agent reports at least this many restarts in the last 24 hours (or since the hub-side reset marker), the hive gets a restart problem chip and drift signal. | | `HIVE_COVERAGE_BADGE_URL` | No | none (the flagship hivecommons/hive gist for org `hivecommons`/`kubestellar` only) | Where the ci-maintainer card's coverage percentage comes from. Either an `http(s)` URL to a shields-style JSON badge (`{"message":"85%"}`) or an SVG badge, or `repo://<ref>/<path>` to read the badge file from the hive's primary repo through the GitHub App client — the form to use for a private repo, e.g. `repo://badges/coverage.svg` for octocov's default layout. Unset on a non-flagship hive, the card shows 0. | | `HIVE_BRANDING_CSS` | No | `<data>/branding/custom.css`, where `<data>` is the parent directory of the configured `data.agents_dir`, falling back to `/data` when that config field is empty — so the shipped default is `/data/branding/custom.css` | Absolute path of the operator override stylesheet the dashboard serves at `/branding/custom.css`. The index document links that URL unconditionally, so a missing file is the normal case and simply 404s. Read **per request**, so an edit takes effect on reload without a restart. The read is guarded (#5854): the file must be a regular file owned by the hive uid or root, not group- or world-writable, at most 128 KiB, and a symlink must resolve inside its own directory — anything else is refused with a logged warning, because whoever can write this path injects CSS into the operator's dashboard. See [Branding a hive, What the code enforces](branding.md#what-the-code-enforces). | diff --git a/src/pkg/hub/selfservice_api.go b/src/pkg/hub/selfservice_api.go new file mode 100644 index 0000000000..899726ca8d --- /dev/null +++ b/src/pkg/hub/selfservice_api.go @@ -0,0 +1,587 @@ +package hub + +// Self-service (byo-) hive registration API (#18060, step 3 of the RFC; the +// store it writes to is selfservice_registry.go, the bearer check is in +// hub_keys.go, and the credential fence is heartbeat_selfservice_fence.go). +// +// A signed-in hub user registers a hive they run on their own infrastructure, +// proves they administer every repo it will work on with THEIR OWN stored +// GitHub token, and receives a random heartbeat key exactly once. They can list +// their own entries, rotate a key, and revoke a hive; a hub admin can revoke +// anyone's. The whole family sits behind HIVE_HUB_SELFSERVICE_REGISTRATION +// (default off) and answers 404 when the flag is off so a hub that does not +// offer self-service does not advertise the routes. GET /status is the one +// exception: it always answers so the dashboard can decide whether the +// Register button opens the form or the self-hosting guide. +// +// Invariants: +// - Every ID this file produces starts with SelfServiceHiveIDPrefix and is +// built as byo-<github login>-<name>; nothing a caller sends can escape it. +// - The raw key is in the 201 and the rotate body and nowhere else: not in +// the list route, not in a log line, not on disk (only its hash is). +// - HIVE_HUB_SECRET is never consulted here and never appears in a response. +// - A caller learns nothing about another user's hive: wrong owner is 404, +// the same answer as a hive that does not exist. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +const ( + // selfServiceRegistrationEnv turns the registration API on. Off by default: + // a hub that does not want to list owner-run hives never exposes it. + selfServiceRegistrationEnv = "HIVE_HUB_SELFSERVICE_REGISTRATION" + // selfServiceMaxPerUserEnv caps how many byo- hives one user may register. + selfServiceMaxPerUserEnv = "HIVE_HUB_SELFSERVICE_MAX_PER_USER" + selfServiceMaxPerUserDefault = 5 + // selfServiceExpiryEnv is how long a registered hive may go without an + // accepted heartbeat before the expiry sweep (a later child of #18060) + // removes it. Read here so the status route can tell the form the window. + selfServiceExpiryEnv = "HIVE_HUB_SELFSERVICE_EXPIRY" + selfServiceExpiryDefault = 24 * time.Hour + + // selfServiceMaxRequestBody bounds a registration body; the largest legal + // body is selfServiceMaxRepos repo refs plus a URL, far under this. + selfServiceMaxRequestBody = 64 << 10 + // selfServiceMaxRepos bounds how many repos one registration may prove + // ownership of, and so how many GitHub calls one request can trigger. + selfServiceMaxRepos = 20 + // selfServiceGitHubAPIBase is the production API the ownership proof + // calls; tests point selfServiceOwnershipChecker.apiBase at a stub. + selfServiceGitHubAPIBase = "https://api.github.com" + // selfServiceOwnershipErrorBodyLimit bounds how much of a GitHub error body + // is read before it is discarded. The body is never logged or returned. + selfServiceOwnershipErrorBodyLimit = 512 + + // selfServiceDashboardURLScheme is the only scheme a dashboard_url may use. + selfServiceDashboardURLScheme = "https" +) + +// selfServiceReservedNamePrefixes are hive ID namespaces a self-service name +// may not borrow. The final ID is always byo-<login>-<name>, so none of these +// could become the ID's prefix; refusing them in the NAME keeps a registration +// from reading like a hosted hive in the dashboard (byo-alice-hosted-prod). +var selfServiceReservedNamePrefixes = []string{"hosted-", "saas-", SelfServiceHiveIDPrefix} + +// selfServiceRefusalReasons are the audit-log reason values for +// selfservice_hive_refused. +const ( + selfServiceRefusedInvalidName = "invalid_name" + selfServiceRefusedReservedPrefix = "reserved_prefix" + selfServiceRefusedDuplicateID = "duplicate_id" + selfServiceRefusedInvalidRepos = "invalid_repos" + selfServiceRefusedInvalidURL = "invalid_dashboard_url" + selfServiceRefusedCap = "per_user_cap" + selfServiceRefusedNoToken = "no_github_token" + selfServiceRefusedNotAdmin = "not_repo_admin" + selfServiceRefusedNoGitHubLogin = "no_github_login" + selfServiceRefusedOwnershipError = "ownership_check_error" +) + +// selfServiceRegistrationEnabled reads the feature flag on every call so an +// operator can flip it without a restart of their test harness; the hub is +// env-configured and reads it at request time like the other hub toggles. +func selfServiceRegistrationEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(selfServiceRegistrationEnv))) { + case "1", "true", "yes", "on": + return true + } + return false +} + +func selfServiceMaxPerUser() int { + return positiveIntEnv(selfServiceMaxPerUserEnv, selfServiceMaxPerUserDefault) +} + +func selfServiceExpiry() time.Duration { + return durationEnv(selfServiceExpiryEnv, selfServiceExpiryDefault) +} + +// selfServiceOwnershipChecker asks GitHub whether a token's user administers +// a repo. It is a struct rather than a function so the production instance can +// own its SSRF-guarded client while a test substitutes an httptest server. +type selfServiceOwnershipChecker struct { + apiBase string + client *http.Client +} + +func defaultSelfServiceOwnershipChecker() *selfServiceOwnershipChecker { + return &selfServiceOwnershipChecker{apiBase: selfServiceGitHubAPIBase, client: liteRepoAccessHTTPClient()} +} + +// ownershipChecker returns the injected checker or the production default. +func (s *HubServer) ownershipChecker() *selfServiceOwnershipChecker { + if s != nil && s.selfServiceOwnership != nil { + return s.selfServiceOwnership + } + return defaultSelfServiceOwnershipChecker() +} + +// repoAdmin reports whether token's user has permissions.admin on owner/repo. +// A 401, 403 or 404 means "no" (GitHub answers 404 for a private repo the +// token cannot see); any other non-2xx is an error, reported by status only so +// nothing GitHub echoes back can reach a log or a response. The RFC requires +// admin specifically; maintain is not accepted (unlike the lite-enrollment +// check, which has different stakes). +func (c *selfServiceOwnershipChecker) repoAdmin(ctx context.Context, token, owner, repo string) (bool, error) { + u, err := url.JoinPath(c.apiBase, "repos", owner, repo) + if err != nil { + return false, fmt.Errorf("build GitHub repo URL: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return false, fmt.Errorf("build GitHub ownership check: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := c.client.Do(req) + if err != nil { + return false, fmt.Errorf("GitHub ownership check failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + switch resp.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound: + return false, nil + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, selfServiceOwnershipErrorBodyLimit)) + return false, fmt.Errorf("GitHub ownership check returned HTTP %d", resp.StatusCode) + } + var payload struct { + Permissions struct { + Admin bool `json:"admin"` + } `json:"permissions"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return false, fmt.Errorf("decode GitHub ownership check: %w", err) + } + return payload.Permissions.Admin, nil +} + +// selfServiceRegisterRequest is the body of POST /api/hub/selfservice/hives. +type selfServiceRegisterRequest struct { + Name string `json:"name"` + Org string `json:"org"` + Repos []string `json:"repos"` + DashboardURL string `json:"dashboard_url"` +} + +// selfServiceHiveView is what the API shows about a registration. It is built +// from SelfServiceHive by hand so a future field on the stored record (a hash, +// a key) can never leak by default. +type selfServiceHiveView struct { + ID string `json:"id"` + Owner string `json:"owner"` + Org string `json:"org,omitempty"` + Repos []string `json:"repos,omitempty"` + DashboardURL string `json:"dashboard_url,omitempty"` + CreatedAt time.Time `json:"created_at"` + OwnershipCheckedAt time.Time `json:"ownership_checked_at,omitempty"` + LastAcceptedHeartbeatAt time.Time `json:"last_accepted_heartbeat_at,omitempty"` + // Online mirrors the registry entry the hive's heartbeats create; false + // until the first accepted heartbeat. + Online bool `json:"online"` +} + +// selfServiceKeySnippets are copy-paste deployment stubs that reference the +// three variables a self-hosted spoke needs. The key is substituted ONCE into +// the response; the templates themselves carry only the variable names. +type selfServiceKeySnippets struct { + DockerCompose string `json:"docker_compose"` + Podman string `json:"podman"` + Kubernetes string `json:"kubernetes"` +} + +// selfServiceSpokeImage is the image the snippets reference. +const selfServiceSpokeImage = "ghcr.io/hivecommons/hive:stable" + +func selfServiceSnippets(hubURL, hiveID, rawKey string) selfServiceKeySnippets { + compose := fmt.Sprintf(`services: + hive: + image: %s + environment: + HIVE_HUB_URL: %q + HIVE_ID: %q + HIVE_HEARTBEAT_KEY: %q +`, selfServiceSpokeImage, hubURL, hiveID, rawKey) + podman := fmt.Sprintf("podman run -d --name hive \\\n -e HIVE_HUB_URL=%q \\\n -e HIVE_ID=%q \\\n -e HIVE_HEARTBEAT_KEY=%q \\\n %s\n", hubURL, hiveID, rawKey, selfServiceSpokeImage) + k8s := fmt.Sprintf(`apiVersion: v1 +kind: Secret +metadata: + name: hive-hub-credentials +type: Opaque +stringData: + HIVE_HUB_URL: %q + HIVE_ID: %q + HIVE_HEARTBEAT_KEY: %q +`, hubURL, hiveID, rawKey) + return selfServiceKeySnippets{DockerCompose: compose, Podman: podman, Kubernetes: k8s} +} + +func selfServiceView(h SelfServiceHive, online bool) selfServiceHiveView { + return selfServiceHiveView{ + ID: h.ID, Owner: h.Owner, Org: h.Org, Repos: h.Repos, DashboardURL: h.DashboardURL, + CreatedAt: h.CreatedAt, OwnershipCheckedAt: h.OwnershipCheckedAt, + LastAcceptedHeartbeatAt: h.LastAcceptedHeartbeatAt, Online: online, + } +} + +// selfServiceOwns reports whether login is the owner of h. Both sides are +// canonicalized so a bare login and its github: form compare equal. +func selfServiceOwns(h SelfServiceHive, login string) bool { + return login != "" && canonicalizeLegacy(h.Owner) == canonicalizeLegacy(login) +} + +// selfServiceCount is how many hives login has registered. +func (s *HubServer) selfServiceCount(login string) int { + n := 0 + for _, h := range s.selfService.list() { + if selfServiceOwns(h, login) { + n++ + } + } + return n +} + +// selfServiceRegistryHas reports whether id is in the general registry. +func (s *HubServer) selfServiceRegistryHas(id string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + for _, h := range s.registry.Hives { + if strings.EqualFold(h.ID, id) { + return true + } + } + return false +} + +func (s *HubServer) selfServiceOnline(id string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + for _, h := range s.registry.Hives { + if h.ID == id { + return h.Online + } + } + return false +} + +// requireSelfServiceEnabled answers 404 when the flag is off so the route +// family is not discoverable on a hub that does not offer it. +func (s *HubServer) requireSelfServiceEnabled(w http.ResponseWriter) bool { + if selfServiceRegistrationEnabled() { + return true + } + writeJSONError(w, http.StatusNotFound, "not found") + return false +} + +func (s *HubServer) auditSelfServiceRefused(login, hiveID, reason string, extra ...any) { + args := append([]any{"user", login, "hive", hiveID, "reason", reason}, extra...) + s.logger.Info("audit: selfservice_hive_refused", args...) +} + +// handleSelfServiceStatus is GET /api/hub/selfservice/status. +func (s *HubServer) handleSelfServiceStatus(w http.ResponseWriter, r *http.Request) { + login := s.getAuthUser(r) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "enabled": selfServiceRegistrationEnabled(), + "max_per_user": selfServiceMaxPerUser(), + "count": s.selfServiceCount(login), + "expiry": selfServiceExpiry().String(), + }) +} + +// handleSelfServiceList is GET /api/hub/selfservice/hives: the caller's own +// registrations, never a hash or a key. +func (s *HubServer) handleSelfServiceList(w http.ResponseWriter, r *http.Request) { + if !s.requireSelfServiceEnabled(w) { + return + } + login := s.getAuthUser(r) + views := make([]selfServiceHiveView, 0) + for _, h := range s.selfService.list() { + if selfServiceOwns(h, login) { + views = append(views, selfServiceView(h, s.selfServiceOnline(h.ID))) + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"hives": views}) +} + +// selfServiceValidateRepos checks every repo ref is owner/name and that there +// is at least one: ownership proof is the whole point, and it needs a repo. +func selfServiceValidateRepos(repos []string) ([]string, error) { + if len(repos) == 0 { + return nil, errors.New("at least one owner/name repo is required") + } + if len(repos) > selfServiceMaxRepos { + return nil, fmt.Errorf("at most %d repos per registration", selfServiceMaxRepos) + } + out := make([]string, 0, len(repos)) + seen := make(map[string]struct{}, len(repos)) + for _, raw := range repos { + repo := strings.TrimSpace(raw) + owner, name, ok := strings.Cut(repo, "/") + if !ok || !isValidRepoRef(repo) || owner == "" || name == "" { + return nil, fmt.Errorf("repo %q must be owner/name", repo) + } + key := strings.ToLower(repo) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + out = append(out, repo) + } + return out, nil +} + +// selfServiceValidateDashboardURL accepts an empty URL or an https URL to a +// public address, the same guard the hive config proxy applies. +func selfServiceValidateDashboardURL(ctx context.Context, raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", nil + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" || u.Hostname() == "" { + return "", errors.New("dashboard_url must be an absolute https URL") + } + if !strings.EqualFold(u.Scheme, selfServiceDashboardURLScheme) { + return "", errors.New("dashboard_url must use https") + } + if u.User != nil { + return "", errors.New("dashboard_url must not carry credentials") + } + if isPrivateURL(ctx, raw) { + return "", errors.New("dashboard_url resolves to a private or internal address") + } + return u.String(), nil +} + +// selfServiceBuildID derives byo-<login>-<name> from the GitHub login and the +// caller's name, refusing names that cannot become a valid hive ID or that +// borrow a reserved namespace. +func selfServiceBuildID(githubLogin, name string) (string, string, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return "", selfServiceRefusedInvalidName, errors.New("name is required") + } + if strings.Contains(name, "/") || !isValidName(name) { + return "", selfServiceRefusedInvalidName, errors.New("name may contain only letters, digits, '.', '_' and '-'") + } + for _, p := range selfServiceReservedNamePrefixes { + if strings.HasPrefix(name, p) { + return "", selfServiceRefusedReservedPrefix, fmt.Errorf("name may not start with the reserved prefix %q", p) + } + } + id := SelfServiceHiveIDPrefix + strings.ToLower(strings.TrimSpace(githubLogin)) + "-" + name + if !isValidName(id) || !IsSelfServiceHiveID(id) { + return "", selfServiceRefusedInvalidName, errors.New("name is too long for a hive ID") + } + return id, "", nil +} + +// handleSelfServiceRegister is POST /api/hub/selfservice/hives. +func (s *HubServer) handleSelfServiceRegister(w http.ResponseWriter, r *http.Request) { + if !s.requireSelfServiceEnabled(w) { + return + } + login := s.getAuthUser(r) + user := loadSaaSUser(login) + if user == nil { + writeJSONError(w, http.StatusUnauthorized, "not authenticated") + return + } + var body selfServiceRegisterRequest + if err := json.NewDecoder(io.LimitReader(r.Body, selfServiceMaxRequestBody)).Decode(&body); err != nil { + writeJSONError(w, http.StatusBadRequest, "invalid JSON body") + return + } + + githubLogin := topRepoProfileLogin(user) + if githubLogin == "" { + s.auditSelfServiceRefused(login, "", selfServiceRefusedNoGitHubLogin) + writeJSONError(w, http.StatusBadRequest, "a GitHub identity is required to register a self-hosted hive") + return + } + id, reason, err := selfServiceBuildID(githubLogin, body.Name) + if err != nil { + s.auditSelfServiceRefused(login, "", reason) + writeJSONError(w, http.StatusBadRequest, err.Error()) + return + } + if s.selfServiceRegistryHas(id) { + s.auditSelfServiceRefused(login, id, selfServiceRefusedDuplicateID) + writeJSONError(w, http.StatusConflict, "a hive with this ID is already registered") + return + } + if _, exists := s.selfService.get(id); exists { + s.auditSelfServiceRefused(login, id, selfServiceRefusedDuplicateID) + writeJSONError(w, http.StatusConflict, "a hive with this ID is already registered") + return + } + org := strings.TrimSpace(body.Org) + if org != "" && !isValidName(org) { + s.auditSelfServiceRefused(login, id, selfServiceRefusedInvalidName) + writeJSONError(w, http.StatusBadRequest, "org may contain only letters, digits, '.', '_' and '-'") + return + } + repos, err := selfServiceValidateRepos(body.Repos) + if err != nil { + s.auditSelfServiceRefused(login, id, selfServiceRefusedInvalidRepos) + writeJSONError(w, http.StatusBadRequest, err.Error()) + return + } + dashboardURL, err := selfServiceValidateDashboardURL(r.Context(), body.DashboardURL) + if err != nil { + s.auditSelfServiceRefused(login, id, selfServiceRefusedInvalidURL) + writeJSONError(w, http.StatusBadRequest, err.Error()) + return + } + if maxPerUser := selfServiceMaxPerUser(); s.selfServiceCount(login) >= maxPerUser { + s.auditSelfServiceRefused(login, id, selfServiceRefusedCap, "max_per_user", maxPerUser) + writeJSONError(w, http.StatusTooManyRequests, fmt.Sprintf("you already have %d self-hosted hives registered; remove one first", maxPerUser)) + return + } + + // Ownership proof with the user's OWN token, never the hub's App token: the + // question is what this person administers, not what the hub can see. + if strings.TrimSpace(user.EncryptedToken) == "" { + s.auditSelfServiceRefused(login, id, selfServiceRefusedNoToken) + writeJSONError(w, http.StatusForbidden, "no GitHub token is stored for your account; sign out and back in with GitHub, then retry") + return + } + token, err := decryptToken(user.EncryptedToken) + if err != nil { + s.logger.Warn("self-service registration: stored token could not be decrypted", "user", login, "error", err) + writeJSONError(w, http.StatusInternalServerError, "stored GitHub token could not be read") + return + } + checker := s.ownershipChecker() + for _, repo := range repos { + owner, name, _ := strings.Cut(repo, "/") + admin, err := checker.repoAdmin(r.Context(), token, owner, name) + if err != nil { + s.auditSelfServiceRefused(login, id, selfServiceRefusedOwnershipError, "repo", repo) + s.logger.Warn("self-service registration: ownership check failed", "user", login, "hive", id, "repo", repo, "error", err) + writeJSONError(w, http.StatusBadGateway, fmt.Sprintf("could not verify your permission on %s; try again", repo)) + return + } + if !admin { + s.auditSelfServiceRefused(login, id, selfServiceRefusedNotAdmin, "repo", repo) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(map[string]string{ + "error": fmt.Sprintf("your GitHub account is not an admin of %s", repo), + "repo": repo, + }) + return + } + } + + now := time.Now().UTC() + rawKey, err := s.selfService.register(SelfServiceHive{ + ID: id, Owner: login, Org: org, Repos: repos, DashboardURL: dashboardURL, OwnershipCheckedAt: now, + }) + if err != nil { + if errors.Is(err, errSelfServiceExists) { + s.auditSelfServiceRefused(login, id, selfServiceRefusedDuplicateID) + writeJSONError(w, http.StatusConflict, "a hive with this ID is already registered") + return + } + s.logger.Error("self-service registration: store write failed", "user", login, "hive", id, "error", err) + writeJSONError(w, http.StatusInternalServerError, "could not persist the registration") + return + } + s.logger.Info("audit: selfservice_hive_registered", "user", login, "hive", id, "repos", len(repos), "org", org) + + stored, _ := s.selfService.get(id) + hubURL := hubPublicURL() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "hive": selfServiceView(stored, false), + "key": rawKey, + "hub_url": hubURL, + "snippets": selfServiceSnippets(hubURL, id, rawKey), + }) +} + +// selfServiceOwnedHive loads the hive named in the path and checks login owns +// it (or, when allowAdmin, that login is a hub admin). A missing hive and +// someone else's hive both answer 404 so the route reveals nothing. +func (s *HubServer) selfServiceOwnedHive(w http.ResponseWriter, r *http.Request, login string, allowAdmin bool) (SelfServiceHive, bool) { + id := strings.ToLower(strings.TrimSpace(r.PathValue("id"))) + h, ok := s.selfService.get(id) + if !ok || !(selfServiceOwns(h, login) || (allowAdmin && isHubAdmin(login))) { + writeJSONError(w, http.StatusNotFound, "self-hosted hive not found") + return SelfServiceHive{}, false + } + return h, true +} + +// handleSelfServiceRotate is POST /api/hub/selfservice/hives/{id}/rotate. +func (s *HubServer) handleSelfServiceRotate(w http.ResponseWriter, r *http.Request) { + if !s.requireSelfServiceEnabled(w) { + return + } + login := s.getAuthUser(r) + h, ok := s.selfServiceOwnedHive(w, r, login, false) + if !ok { + return + } + rawKey, err := s.selfService.rotate(h.ID) + if err != nil { + if errors.Is(err, errSelfServiceNotFound) { + writeJSONError(w, http.StatusNotFound, "self-hosted hive not found") + return + } + s.logger.Error("self-service rotation: store write failed", "user", login, "hive", h.ID, "error", err) + writeJSONError(w, http.StatusInternalServerError, "could not persist the new key") + return + } + s.logger.Info("audit: selfservice_hive_rotated", "user", login, "hive", h.ID, "overlap", selfServiceKeyRotationOverlap.String()) + hubURL := hubPublicURL() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": h.ID, + "key": rawKey, + "hub_url": hubURL, + "overlap": selfServiceKeyRotationOverlap.String(), + "snippets": selfServiceSnippets(hubURL, h.ID, rawKey), + }) +} + +// handleSelfServiceDelete is DELETE /api/hub/selfservice/hives/{id}: the +// owner or a hub admin revokes the key and removes the registry entry. +func (s *HubServer) handleSelfServiceDelete(w http.ResponseWriter, r *http.Request) { + if !s.requireSelfServiceEnabled(w) { + return + } + login := s.getAuthUser(r) + h, ok := s.selfServiceOwnedHive(w, r, login, true) + if !ok { + return + } + removed, err := s.selfService.revoke(h.ID) + if err != nil { + s.logger.Error("self-service revocation: store write failed", "user", login, "hive", h.ID, "error", err) + writeJSONError(w, http.StatusInternalServerError, "could not persist the revocation") + return + } + s.removeRegistryEntry(h.ID, login) + s.logger.Info("audit: selfservice_hive_revoked", "user", login, "hive", h.ID, "owner", h.Owner, "by_admin", !selfServiceOwns(h, login)) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"removed": removed, "id": h.ID}) +} diff --git a/src/pkg/hub/selfservice_api_test.go b/src/pkg/hub/selfservice_api_test.go new file mode 100644 index 0000000000..ad82866985 --- /dev/null +++ b/src/pkg/hub/selfservice_api_test.go @@ -0,0 +1,618 @@ +package hub + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const ( + selfServiceTestOrigin = "https://hive.hivecommons.dev" + selfServiceTestUser = "alice" + selfServiceTestOther = "bob" + selfServiceTestAdmin = "root-admin" + selfServiceTestToken = "ghp_selfservice_test_token_do_not_leak" + selfServiceTestAdminEnv = "HIVE_HUB_ADMINS" +) + +// selfServiceTestEnv is a HubServer wired for the registration API with a +// stubbed GitHub, a capturing logger, temp stores and the flag on. +type selfServiceTestEnv struct { + s *HubServer + logs *lockedBuffer + github *httptest.Server + // adminRepos answers permissions.admin=true; any other repo under the + // stub owner answers admin=false; unknown repos answer 404. + adminRepos map[string]bool + now *time.Time +} + +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func newSelfServiceTestEnv(t *testing.T) *selfServiceTestEnv { + t.Helper() + t.Setenv(selfServiceRegistrationEnv, "true") + t.Setenv(selfServiceMaxPerUserEnv, "") + t.Setenv(selfServiceTestAdminEnv, selfServiceTestAdmin) + withTempSaaSDirs(t) + + dir := t.TempDir() + origHMAC := hmacKeyPath + hmacKeyPath = filepath.Join(dir, "hmac.key") + t.Cleanup(func() { hmacKeyPath = origHMAC }) + + origResolver := privateURLResolver + privateURLResolver = func(ctx context.Context, host string) ([]string, error) { + return []string{"203.0.113.10"}, nil + } + t.Cleanup(func() { privateURLResolver = origResolver }) + + env := &selfServiceTestEnv{logs: &lockedBuffer{}, adminRepos: map[string]bool{"alice/lab": true, "alice/tools": true, "alice/readonly": false}} + env.github = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+selfServiceTestToken { + w.WriteHeader(http.StatusUnauthorized) + return + } + repo := strings.TrimPrefix(r.URL.Path, "/repos/") + admin, known := env.adminRepos[repo] + if !known { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"full_name": repo, "permissions": map[string]bool{"admin": admin, "push": true}}) + })) + t.Cleanup(env.github.Close) + + logger := slog.New(slog.NewTextHandler(env.logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + prevDefault := slog.Default() + slog.SetDefault(logger) + t.Cleanup(func() { slog.SetDefault(prevDefault) }) + + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + env.now = &now + store := newSelfServiceStore(filepath.Join(dir, "hub-selfservice.json")) + store.now = func() time.Time { return now } + + env.s = &HubServer{ + logger: logger, + hubSecret: grantableTestSecret, + registryPath: filepath.Join(dir, "hub-registry.json"), + selfService: store, + selfServiceOwnership: &selfServiceOwnershipChecker{apiBase: env.github.URL, client: env.github.Client()}, + registry: Registry{Hives: []RegistryEntry{}}, + } + + encrypted, err := encryptToken(selfServiceTestToken) + if err != nil { + t.Fatalf("encryptToken: %v", err) + } + for _, u := range []*SaaSUser{ + {GitHubUsername: selfServiceTestUser, EncryptedToken: encrypted}, + {GitHubUsername: selfServiceTestOther, EncryptedToken: encrypted}, + {GitHubUsername: selfServiceTestAdmin, EncryptedToken: encrypted}, + } { + if err := saveSaaSUser(u); err != nil { + t.Fatalf("saveSaaSUser(%s): %v", u.GitHubUsername, err) + } + } + return env +} + +// request builds a request through requireAuth for the given handler, signed +// in as user (empty user = anonymous). The Origin header satisfies the CSRF +// check for writes, as a browser on the hub origin would. +func (env *selfServiceTestEnv) request(t *testing.T, user, method, path, body string, handler http.HandlerFunc) *httptest.ResponseRecorder { + t.Helper() + var rd io.Reader + if body != "" { + rd = strings.NewReader(body) + } + req := httptest.NewRequest(method, path, rd) + req.Header.Set("Origin", selfServiceTestOrigin) + if user != "" { + req.AddCookie(&http.Cookie{ + Name: "hive_hub_user", + Value: mintHubUserCookieValueV2(deriveDomainKey(grantableTestSecret, infoSessionEd25519Seed), user), + }) + } + // Path values are set by the mux in production; the tests call the + // handler directly, so they are filled in from the path here. + if rest, ok := strings.CutPrefix(path, "/api/hub/selfservice/hives/"); ok { + id, _, _ := strings.Cut(rest, "/") + req.SetPathValue("id", id) + } + rec := httptest.NewRecorder() + env.s.requireAuth(handler)(rec, req) + return rec +} + +func (env *selfServiceTestEnv) register(t *testing.T, user, body string) *httptest.ResponseRecorder { + t.Helper() + return env.request(t, user, http.MethodPost, "/api/hub/selfservice/hives", body, env.s.handleSelfServiceRegister) +} + +func (env *selfServiceTestEnv) list(t *testing.T, user string) *httptest.ResponseRecorder { + t.Helper() + return env.request(t, user, http.MethodGet, "/api/hub/selfservice/hives", "", env.s.handleSelfServiceList) +} + +func (env *selfServiceTestEnv) rotate(t *testing.T, user, id string) *httptest.ResponseRecorder { + t.Helper() + return env.request(t, user, http.MethodPost, "/api/hub/selfservice/hives/"+id+"/rotate", "", env.s.handleSelfServiceRotate) +} + +func (env *selfServiceTestEnv) remove(t *testing.T, user, id string) *httptest.ResponseRecorder { + t.Helper() + return env.request(t, user, http.MethodDelete, "/api/hub/selfservice/hives/"+id, "", env.s.handleSelfServiceDelete) +} + +func (env *selfServiceTestEnv) status(t *testing.T, user string) *httptest.ResponseRecorder { + t.Helper() + return env.request(t, user, http.MethodGet, "/api/hub/selfservice/status", "", env.s.handleSelfServiceStatus) +} + +const selfServiceHappyBody = `{"name":"Lab","org":"alice","repos":["alice/lab","alice/tools"],"dashboard_url":"https://lab.example.org/"}` + +// registerHappy registers alice's lab hive and returns the ID and raw key. +func (env *selfServiceTestEnv) registerHappy(t *testing.T) (string, string) { + t.Helper() + rec := env.register(t, selfServiceTestUser, selfServiceHappyBody) + if rec.Code != http.StatusCreated { + t.Fatalf("register: status %d body %s", rec.Code, rec.Body.String()) + } + var resp struct { + Hive selfServiceHiveView `json:"hive"` + Key string `json:"key"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode register response: %v", err) + } + if resp.Key == "" || resp.Hive.ID == "" { + t.Fatalf("register response missing id or key: %s", rec.Body.String()) + } + return resp.Hive.ID, resp.Key +} + +func decodeError(t *testing.T, rec *httptest.ResponseRecorder) map[string]string { + t.Helper() + var m map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &m); err != nil { + t.Fatalf("error body is not JSON: %q", rec.Body.String()) + } + return m +} + +func TestSelfServiceAPIUnauthenticated(t *testing.T) { + env := newSelfServiceTestEnv(t) + cases := map[string]*httptest.ResponseRecorder{ + "status": env.status(t, ""), + "list": env.list(t, ""), + "register": env.register(t, "", selfServiceHappyBody), + "rotate": env.rotate(t, "", "byo-alice-lab"), + "delete": env.remove(t, "", "byo-alice-lab"), + } + for name, rec := range cases { + if rec.Code != http.StatusUnauthorized { + t.Errorf("%s: anonymous status = %d, want 401 (body %s)", name, rec.Code, rec.Body.String()) + } + } + if len(env.s.selfService.list()) != 0 { + t.Fatal("anonymous request stored something") + } +} + +func TestSelfServiceAPIFlagOffIsNotFound(t *testing.T) { + env := newSelfServiceTestEnv(t) + t.Setenv(selfServiceRegistrationEnv, "") + cases := map[string]*httptest.ResponseRecorder{ + "list": env.list(t, selfServiceTestUser), + "register": env.register(t, selfServiceTestUser, selfServiceHappyBody), + "rotate": env.rotate(t, selfServiceTestUser, "byo-alice-lab"), + "delete": env.remove(t, selfServiceTestUser, "byo-alice-lab"), + } + for name, rec := range cases { + if rec.Code != http.StatusNotFound { + t.Errorf("%s with flag off: status = %d, want 404", name, rec.Code) + } + } + if len(env.s.selfService.list()) != 0 { + t.Fatal("flag-off register stored something") + } + rec := env.status(t, selfServiceTestUser) + if rec.Code != http.StatusOK { + t.Fatalf("status with flag off: %d", rec.Code) + } + var st struct { + Enabled bool `json:"enabled"` + MaxPerUser int `json:"max_per_user"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if st.Enabled || st.MaxPerUser != selfServiceMaxPerUserDefault { + t.Fatalf("status = %+v, want enabled=false max=%d", st, selfServiceMaxPerUserDefault) + } +} + +func TestSelfServiceAPIRegisterHappyPath(t *testing.T) { + env := newSelfServiceTestEnv(t) + rec := env.register(t, selfServiceTestUser, selfServiceHappyBody) + if rec.Code != http.StatusCreated { + t.Fatalf("status %d body %s", rec.Code, rec.Body.String()) + } + var resp struct { + Hive selfServiceHiveView `json:"hive"` + Key string `json:"key"` + HubURL string `json:"hub_url"` + Snippets selfServiceKeySnippets `json:"snippets"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Hive.ID != "byo-alice-lab" { + t.Fatalf("id = %q, want byo-alice-lab", resp.Hive.ID) + } + if resp.Hive.Owner != selfServiceTestUser || resp.Hive.Org != "alice" || len(resp.Hive.Repos) != 2 || resp.Hive.DashboardURL != "https://lab.example.org/" { + t.Fatalf("view = %+v", resp.Hive) + } + if resp.Hive.OwnershipCheckedAt.IsZero() { + t.Fatal("OwnershipCheckedAt not stamped") + } + for name, snippet := range map[string]string{"compose": resp.Snippets.DockerCompose, "podman": resp.Snippets.Podman, "k8s": resp.Snippets.Kubernetes} { + for _, want := range []string{"HIVE_HUB_URL", "HIVE_ID", "HIVE_HEARTBEAT_KEY", resp.Key, resp.Hive.ID} { + if !strings.Contains(snippet, want) { + t.Errorf("%s snippet lacks %q", name, want) + } + } + } + + // The store holds only the hash, and the hub accepts the raw key as this + // hive's heartbeat bearer and no other hive's. + stored, ok := env.s.selfService.get("byo-alice-lab") + if !ok { + t.Fatal("hive not stored") + } + if stored.KeyHash != hashSelfServiceKey(resp.Key) || stored.KeyHash == resp.Key { + t.Fatal("store does not hold exactly the hash of the returned key") + } + if gen, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, "byo-alice-lab"); !ok || gen != selfServiceBearerGeneration { + t.Fatalf("raw key does not authenticate the hive: gen=%d ok=%v", gen, ok) + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, "byo-alice-other"); ok { + t.Fatal("raw key authenticated a different hive") + } + if !strings.Contains(env.logs.String(), "audit: selfservice_hive_registered") { + t.Fatal("no registration audit line") + } + + // The GitHub stub saw the user's own token, not the hub master. + listRec := env.list(t, selfServiceTestUser) + if listRec.Code != http.StatusOK { + t.Fatalf("list: %d", listRec.Code) + } + var listed struct { + Hives []selfServiceHiveView `json:"hives"` + } + if err := json.Unmarshal(listRec.Body.Bytes(), &listed); err != nil { + t.Fatal(err) + } + if len(listed.Hives) != 1 || listed.Hives[0].ID != "byo-alice-lab" { + t.Fatalf("list = %+v", listed.Hives) + } + for _, forbidden := range []string{resp.Key, stored.KeyHash, "key_hash", grantableTestSecret} { + if strings.Contains(listRec.Body.String(), forbidden) { + t.Fatalf("list body leaks %q", forbidden) + } + } + // The other user sees nothing. + otherRec := env.list(t, selfServiceTestOther) + if !strings.Contains(otherRec.Body.String(), `"hives":[]`) { + t.Fatalf("other user's list = %s", otherRec.Body.String()) + } +} + +func TestSelfServiceAPIRegisterRefusals(t *testing.T) { + env := newSelfServiceTestEnv(t) + // Pre-existing registry entry for the duplicate case. + env.s.registry.Hives = []RegistryEntry{{ID: "byo-alice-taken"}} + cases := []struct { + name string + body string + status int + reason string + }{ + {"reserved hosted prefix", `{"name":"hosted-prod","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedReservedPrefix}, + {"reserved saas prefix", `{"name":"saas-prod","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedReservedPrefix}, + {"reserved byo prefix", `{"name":"byo-x","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedReservedPrefix}, + {"slash in name", `{"name":"a/b","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedInvalidName}, + {"invalid characters", `{"name":"lab!","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedInvalidName}, + {"empty name", `{"name":"","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedInvalidName}, + {"too long", fmt.Sprintf(`{"name":%q,"repos":["alice/lab"]}`, strings.Repeat("x", 120)), http.StatusBadRequest, selfServiceRefusedInvalidName}, + {"duplicate registry id", `{"name":"taken","repos":["alice/lab"]}`, http.StatusConflict, selfServiceRefusedDuplicateID}, + {"no repos", `{"name":"lab","repos":[]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, + {"bare org repo", `{"name":"lab","repos":["alice"]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, + {"http dashboard", `{"name":"lab","repos":["alice/lab"],"dashboard_url":"http://lab.example.org"}`, http.StatusBadRequest, selfServiceRefusedInvalidURL}, + {"private dashboard", `{"name":"lab","repos":["alice/lab"],"dashboard_url":"https://10.0.0.5/"}`, http.StatusBadRequest, selfServiceRefusedInvalidURL}, + {"bad json", `{`, http.StatusBadRequest, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := env.register(t, selfServiceTestUser, tc.body) + if rec.Code != tc.status { + t.Fatalf("status = %d, want %d (body %s)", rec.Code, tc.status, rec.Body.String()) + } + if tc.reason != "" && !strings.Contains(env.logs.String(), "reason="+tc.reason) { + t.Fatalf("no refused audit line with reason %s", tc.reason) + } + }) + } + if n := len(env.s.selfService.list()); n != 0 { + t.Fatalf("refused registrations stored %d hives", n) + } + // The duplicate check also covers the self-service store itself. + env.registerHappy(t) + rec := env.register(t, selfServiceTestUser, selfServiceHappyBody) + if rec.Code != http.StatusConflict { + t.Fatalf("second identical registration: %d", rec.Code) + } +} + +func TestSelfServiceAPIOwnershipProof(t *testing.T) { + env := newSelfServiceTestEnv(t) + t.Run("admin false names the repo", func(t *testing.T) { + rec := env.register(t, selfServiceTestUser, `{"name":"lab","repos":["alice/lab","alice/readonly"]}`) + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d body %s", rec.Code, rec.Body.String()) + } + body := decodeError(t, rec) + if body["repo"] != "alice/readonly" || !strings.Contains(body["error"], "alice/readonly") { + t.Fatalf("refusal does not name the repo: %v", body) + } + if !strings.Contains(env.logs.String(), "reason="+selfServiceRefusedNotAdmin) { + t.Fatal("no refused audit line") + } + }) + t.Run("unknown repo refused", func(t *testing.T) { + rec := env.register(t, selfServiceTestUser, `{"name":"lab","repos":["alice/missing"]}`) + if rec.Code != http.StatusForbidden || decodeError(t, rec)["repo"] != "alice/missing" { + t.Fatalf("status = %d body %s", rec.Code, rec.Body.String()) + } + }) + t.Run("no stored token", func(t *testing.T) { + if err := saveSaaSUser(&SaaSUser{GitHubUsername: selfServiceTestOther}); err != nil { + t.Fatal(err) + } + rec := env.register(t, selfServiceTestOther, `{"name":"lab","repos":["alice/lab"]}`) + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d body %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(env.logs.String(), "reason="+selfServiceRefusedNoToken) { + t.Fatal("no refused audit line") + } + }) + if n := len(env.s.selfService.list()); n != 0 { + t.Fatalf("ownership failures stored %d hives", n) + } + // Admin on every repo passes. + env.registerHappy(t) +} + +func TestSelfServiceAPIPerUserCap(t *testing.T) { + env := newSelfServiceTestEnv(t) + t.Setenv(selfServiceMaxPerUserEnv, "2") + for _, name := range []string{"one", "two"} { + rec := env.register(t, selfServiceTestUser, fmt.Sprintf(`{"name":%q,"repos":["alice/lab"]}`, name)) + if rec.Code != http.StatusCreated { + t.Fatalf("register %s: %d %s", name, rec.Code, rec.Body.String()) + } + } + rec := env.register(t, selfServiceTestUser, `{"name":"three","repos":["alice/lab"]}`) + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("over cap: status = %d, want 429", rec.Code) + } + if n := len(env.s.selfService.list()); n != 2 { + t.Fatalf("store has %d hives after cap, want 2", n) + } + if !strings.Contains(env.logs.String(), "reason="+selfServiceRefusedCap) { + t.Fatal("no cap audit line") + } + // The cap is per user: bob can still register. + rec = env.register(t, selfServiceTestOther, `{"name":"mine","repos":["alice/lab"]}`) + if rec.Code != http.StatusCreated { + t.Fatalf("other user under cap: %d %s", rec.Code, rec.Body.String()) + } + var st struct { + Count int `json:"count"` + } + if err := json.Unmarshal(env.status(t, selfServiceTestUser).Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if st.Count != 2 { + t.Fatalf("status count = %d, want 2", st.Count) + } +} + +func TestSelfServiceAPIRotateInvalidatesOldKey(t *testing.T) { + env := newSelfServiceTestEnv(t) + id, oldKey := env.registerHappy(t) + + rec := env.rotate(t, selfServiceTestUser, id) + if rec.Code != http.StatusOK { + t.Fatalf("rotate: %d %s", rec.Code, rec.Body.String()) + } + var resp struct { + ID string `json:"id"` + Key string `json:"key"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.ID != id || resp.Key == "" || resp.Key == oldKey { + t.Fatalf("rotate response = %+v", resp) + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, id); !ok { + t.Fatal("new key does not authenticate") + } + // Inside the overlap the old key still works so a redeploy is not an + // outage; after it, the old key is dead. + if _, ok := env.s.verifyHeartbeatBearerGeneration(oldKey, id); !ok { + t.Fatal("old key refused inside the rotation overlap") + } + *env.now = env.now.Add(selfServiceKeyRotationOverlap + time.Second) + if _, ok := env.s.verifyHeartbeatBearerGeneration(oldKey, id); ok { + t.Fatal("old key still authenticates after the overlap") + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, id); !ok { + t.Fatal("new key stopped authenticating") + } + if !strings.Contains(env.logs.String(), "audit: selfservice_hive_rotated") { + t.Fatal("no rotation audit line") + } +} + +func TestSelfServiceAPIDeleteRemoves(t *testing.T) { + env := newSelfServiceTestEnv(t) + id, key := env.registerHappy(t) + // A heartbeat created a registry entry; delete must take it too. + env.s.registry.Hives = append(env.s.registry.Hives, RegistryEntry{ID: id, Online: true}) + + rec := env.remove(t, selfServiceTestUser, id) + if rec.Code != http.StatusOK { + t.Fatalf("delete: %d %s", rec.Code, rec.Body.String()) + } + if _, ok := env.s.selfService.get(id); ok { + t.Fatal("hive still in the self-service store") + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(key, id); ok { + t.Fatal("revoked key still authenticates") + } + if env.s.selfServiceRegistryHas(id) { + t.Fatal("registry entry survived the delete") + } + if !strings.Contains(env.logs.String(), "audit: selfservice_hive_revoked") { + t.Fatal("no revocation audit line") + } + // Gone means gone: a second delete is 404. + if rec := env.remove(t, selfServiceTestUser, id); rec.Code != http.StatusNotFound { + t.Fatalf("second delete: %d", rec.Code) + } +} + +func TestSelfServiceAPIAnotherUsersHiveIsNotFound(t *testing.T) { + env := newSelfServiceTestEnv(t) + id, key := env.registerHappy(t) + + if rec := env.rotate(t, selfServiceTestOther, id); rec.Code != http.StatusNotFound { + t.Fatalf("rotate by other user: %d, want 404", rec.Code) + } + if rec := env.remove(t, selfServiceTestOther, id); rec.Code != http.StatusNotFound { + t.Fatalf("delete by other user: %d, want 404", rec.Code) + } + if rec := env.rotate(t, selfServiceTestUser, "byo-alice-nope"); rec.Code != http.StatusNotFound { + t.Fatalf("rotate of missing hive: %d, want 404", rec.Code) + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(key, id); !ok { + t.Fatal("other user's attempts disturbed the owner's key") + } + // A hub admin may revoke but not rotate (rotation hands out the key). + if rec := env.rotate(t, selfServiceTestAdmin, id); rec.Code != http.StatusNotFound { + t.Fatalf("rotate by admin: %d, want 404", rec.Code) + } + if rec := env.remove(t, selfServiceTestAdmin, id); rec.Code != http.StatusOK { + t.Fatalf("delete by admin: %d %s", rec.Code, rec.Body.String()) + } + if _, ok := env.s.selfService.get(id); ok { + t.Fatal("admin delete did not remove the hive") + } + if !strings.Contains(env.logs.String(), "by_admin=true") { + t.Fatal("admin revocation not attributed in the audit line") + } +} + +func TestSelfServiceAPINeverLeaksKeyMaterial(t *testing.T) { + env := newSelfServiceTestEnv(t) + id, key := env.registerHappy(t) + rec := env.rotate(t, selfServiceTestUser, id) + var rotated struct { + Key string `json:"key"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &rotated); err != nil { + t.Fatal(err) + } + env.list(t, selfServiceTestUser) + env.status(t, selfServiceTestUser) + env.remove(t, selfServiceTestOther, id) + + logs := env.logs.String() + for _, secret := range []string{key, rotated.Key, grantableTestSecret, selfServiceTestToken} { + if secret == "" { + t.Fatal("empty secret under test") + } + if strings.Contains(logs, secret) { + t.Fatalf("log output contains secret material: %s", logs) + } + } + data, err := os.ReadFile(env.s.selfService.path) + if err != nil { + t.Fatalf("read store: %v", err) + } + disk := string(data) + for _, secret := range []string{key, rotated.Key, grantableTestSecret, selfServiceTestToken} { + if strings.Contains(disk, secret) { + t.Fatal("store file contains raw secret material") + } + } + if !strings.Contains(disk, hashSelfServiceKey(rotated.Key)) || !strings.Contains(disk, hashSelfServiceKey(key)) { + t.Fatal("store file lacks the expected hashes") + } + // The GitHub client sent the user's token, never the hub master or a key: + // the stub answered 401 to anything else, and registration succeeded. +} + +func TestSelfServiceRegistrationEnabledParsing(t *testing.T) { + for raw, want := range map[string]bool{"": false, "0": false, "false": false, "no": false, "1": true, "true": true, "YES": true, "On": true, "maybe": false} { + t.Setenv(selfServiceRegistrationEnv, raw) + if got := selfServiceRegistrationEnabled(); got != want { + t.Errorf("%q: enabled = %v, want %v", raw, got, want) + } + } + t.Setenv(selfServiceMaxPerUserEnv, "") + if selfServiceMaxPerUser() != selfServiceMaxPerUserDefault { + t.Fatal("default cap not applied") + } + t.Setenv(selfServiceExpiryEnv, "") + if selfServiceExpiry() != selfServiceExpiryDefault { + t.Fatal("default expiry not applied") + } + t.Setenv(selfServiceExpiryEnv, "48h") + if selfServiceExpiry() != 2*selfServiceExpiryDefault { + t.Fatal("expiry env override ignored") + } +} diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index b0d696d560..bf9a127c89 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1048,9 +1048,13 @@ type HubServer struct { // selfService holds owner-registered (byo-) hives and the hashes of their // random heartbeat keys (#18060). Nil on bare test literals; every method // is nil-safe and answers "nothing registered". - selfService *selfServiceStore - myHivesCacheMu sync.Mutex - myHivesCache map[string]myHivesSnapshot + selfService *selfServiceStore + // selfServiceOwnership is the GitHub repo-admin check the registration API + // runs with the user's own token (selfservice_api.go). Nil means the + // production api.github.com checker; tests inject an httptest stub. + selfServiceOwnership *selfServiceOwnershipChecker + myHivesCacheMu sync.Mutex + myHivesCache map[string]myHivesSnapshot // keyGenerations is the ordered set of master generations this hub accepts // (hub_generations.go). Until a rotation happens it holds exactly ONE // generation whose secret IS hubSecret, so every derived key is @@ -1773,6 +1777,14 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub // no hub decision reads a chain. See src/docs/delegation-chain.md. s.mux.HandleFunc("GET "+delegation.KeysPath, s.handleDelegationKeys) s.mux.HandleFunc("DELETE /api/hub/registry/{id}", s.handleRegistryDelete) + // Self-service (byo-) hive registration (#18060, selfservice_api.go). + // Behind HIVE_HUB_SELFSERVICE_REGISTRATION: every route but /status answers + // 404 while the flag is off. + s.mux.HandleFunc("GET /api/hub/selfservice/status", s.requireAuth(s.handleSelfServiceStatus)) + s.mux.HandleFunc("GET /api/hub/selfservice/hives", s.requireAuth(s.handleSelfServiceList)) + s.mux.HandleFunc("POST /api/hub/selfservice/hives", s.requireAuth(s.handleSelfServiceRegister)) + s.mux.HandleFunc("POST /api/hub/selfservice/hives/{id}/rotate", s.requireAuth(s.handleSelfServiceRotate)) + s.mux.HandleFunc("DELETE /api/hub/selfservice/hives/{id}", s.requireAuth(s.handleSelfServiceDelete)) s.mux.HandleFunc("POST /api/contribute/register", s.handleContributeProxy) s.mux.HandleFunc("GET /api/contribute/status", s.handleContributeStatus) s.mux.HandleFunc("GET /api/contribute/ws", s.handleContributeWSProxy) From 970f0d7262d201ec69977ec62993021709f82607 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 19:54:33 -0500 Subject: [PATCH 194/223] =?UTF-8?q?=E2=9C=A8=20refderive:=20intake.referen?= =?UTF-8?q?ceRepoFields=20hook=20and=20a=20pure=20parser=20deriving=20refe?= =?UTF-8?q?rence=20repos=20from=20an=20accepted=20request=20issue=20(#1816?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workflow models gain spec.intake.referenceRepoFields / outOfScopeFields, config.ReferenceRepo gains a never-serialized DerivedFrom provenance with Derived(), and the new pkg/refderive parser turns an accepted assessment request issue (GitHub issue-form markdown) into write-tier reference repo entries, refusing with a typed reason when the request is closed, not triage/accepted, or accepted by a bot or a user without write access. Fixes #18157 Part of #18139 Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...8139-reference-repos-from-request-issue.md | 1 + src/pkg/config/reference_repos.go | 11 + src/pkg/config/reference_repos_test.go | 34 ++ src/pkg/refderive/derive.go | 344 ++++++++++++++++++ src/pkg/refderive/derive_test.go | 294 +++++++++++++++ src/pkg/refderive/testdata/techdocs-395.md | 52 +++ src/pkg/workflowmodel/model.go | 21 ++ 7 files changed, 757 insertions(+) create mode 100644 changelog.d/added-18139-reference-repos-from-request-issue.md create mode 100644 src/pkg/refderive/derive.go create mode 100644 src/pkg/refderive/derive_test.go create mode 100644 src/pkg/refderive/testdata/techdocs-395.md diff --git a/changelog.d/added-18139-reference-repos-from-request-issue.md b/changelog.d/added-18139-reference-repos-from-request-issue.md new file mode 100644 index 0000000000..4c29cedc99 --- /dev/null +++ b/changelog.d/added-18139-reference-repos-from-request-issue.md @@ -0,0 +1 @@ +- Workflow models gain `spec.intake.referenceRepoFields` / `outOfScopeFields`, `config.ReferenceRepo` gains a never-serialized `DerivedFrom` provenance, and the new `pkg/refderive` parser derives write-tier reference repos from an accepted assessment request issue, refusing with a typed reason when the request is closed, not `triage/accepted`, or accepted by a bot or a user without write access (#18139, #18157). diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go index bb5093ebeb..1d36d47265 100644 --- a/src/pkg/config/reference_repos.go +++ b/src/pkg/config/reference_repos.go @@ -38,8 +38,19 @@ type ReferenceRepo struct { // authorized for Repo, used instead of minting from the App installation // on the target org. Only meaningful with Write. WriteTokenFile string `yaml:"write_token_file,omitempty" json:"write_token_file,omitempty"` + // DerivedFrom is the "owner/repo#N" of the accepted request issue this + // entry was derived from (hivecommons/hive#18139), empty for an entry an + // operator wrote. It is deliberately excluded from YAML: a derived entry + // is rebuilt from the issue on every config load and must never land in + // an overlay or the runtime config, where it would outlive the grant + // (see hivecommons/hive#18137). The dashboard reads it as JSON. + DerivedFrom string `yaml:"-" json:"derived_from,omitempty"` } +// Derived reports whether the entry came from an accepted request issue +// rather than from the operator's config. +func (r ReferenceRepo) Derived() bool { return r.DerivedFrom != "" } + const ( ReferenceWriteIssues = "issues" ReferenceWritePRs = "prs" diff --git a/src/pkg/config/reference_repos_test.go b/src/pkg/config/reference_repos_test.go index 558e4674b4..fce0389522 100644 --- a/src/pkg/config/reference_repos_test.go +++ b/src/pkg/config/reference_repos_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "strings" "testing" @@ -150,3 +151,36 @@ func TestReferenceRepoTierEmptyAndNilGuards(t *testing.T) { t.Fatalf("slugs: %v", got) } } + +func TestReferenceRepoDerivedFromNeverSerializesToYAML(t *testing.T) { + entries := []ReferenceRepo{ + {Repo: "hivecommons/docs", Write: []string{ReferenceWriteIssues, ReferenceWritePRs}, DerivedFrom: "cncf/techdocs#395"}, + {Repo: "hivecommons/hive", DerivedFrom: "cncf/techdocs#395"}, + } + if !entries[0].Derived() || (ReferenceRepo{Repo: "x/y"}).Derived() { + t.Fatal("Derived must follow DerivedFrom") + } + out, err := yaml.Marshal(entries) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "395") || strings.Contains(string(out), "derived") { + t.Fatalf("derived provenance leaked into YAML:\n%s", out) + } + var back []ReferenceRepo + if err := yaml.Unmarshal(out, &back); err != nil { + t.Fatal(err) + } + for _, r := range back { + if r.Derived() { + t.Fatalf("round-tripped entry must not read as derived: %+v", r) + } + } + js, err := json.Marshal(entries[0]) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(js), `"derived_from":"cncf/techdocs#395"`) { + t.Fatalf("JSON must carry derived_from for the dashboard:\n%s", js) + } +} diff --git a/src/pkg/refderive/derive.go b/src/pkg/refderive/derive.go new file mode 100644 index 0000000000..a58651c8fb --- /dev/null +++ b/src/pkg/refderive/derive.go @@ -0,0 +1,344 @@ +// Package refderive turns an accepted assessment request issue into derived +// project.reference_repos entries (hivecommons/hive#18139). +// +// The motivating case is the CNCF TechDocs hive: its authorized repo is +// cncf/techdocs, and an assessment request issue there (cncf/techdocs#395) +// names the requesting project's repos in the issue-form fields "Website or +// documentation repository" and "Other repositories or sites in scope". Once +// a maintainer applies triage/accepted, the hive should be able to open and +// update issues and PRs in those repos without anyone editing config. +// +// This package is the pure half: it reads facts the caller has already +// fetched (the issue, who applied the grant label and with what permission) +// and the bound workflow model's intake declaration, and returns the entries +// with their provenance, or a typed reason for refusing. It never calls +// GitHub, never touches config files and never widens an entry past issues +// and PRs. Fetching the issue, checking the App installation on the +// requesting org and merging the entries into the running config are the +// later children of the RFC. +package refderive + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// GrantLabel is the label whose presence, applied by a user with write +// access to the authorized repo, turns a request issue into a grant. +const GrantLabel = "triage/accepted" + +// DefaultHost is the forge host repository URLs are read from when Options +// names none. The hive is single-host, so a URL on any other host is text. +const DefaultHost = "github.com" + +// DerivedWrite is the fixed tier of every derived entry: open issues and +// PRs, comment on the hive's own items. A derived entry can never be wider. +var DerivedWrite = []string{config.ReferenceWriteIssues, config.ReferenceWritePRs} + +// writePermissions are the repository permission levels GitHub reports for +// a user who may push: the applier of GrantLabel must hold one of them. +var writePermissions = map[string]bool{"admin": true, "maintain": true, "write": true} + +// noResponse is what a GitHub issue form renders for an empty optional field. +const noResponse = "_No response_" + +// headingPrefix opens an issue-form field in the rendered body. +const headingPrefix = "### " + +// Refusal reasons. A caller that gets one leaves the hive's config untouched +// and logs the reason; none of them is a bug. +var ( + // ErrNoIntakeFields is returned when the bound model declares no + // intake.referenceRepoFields: the hive is not meant to derive anything. + ErrNoIntakeFields = errors.New("workflow model declares no intake.referenceRepoFields") + // ErrClosed is returned for a closed request: the grant ended with it. + ErrClosed = errors.New("request issue is closed") + // ErrNotAccepted is returned when GrantLabel is not on the issue. + ErrNotAccepted = errors.New("request issue does not carry " + GrantLabel) + // ErrAccepterUnknown is returned when the caller could not tell who + // applied GrantLabel; an unattributed grant is no grant. + ErrAccepterUnknown = errors.New("cannot tell who applied " + GrantLabel) + // ErrAccepterNoWrite is returned when the applier lacks write access to + // the authorized repo: anyone with triage can move labels, only a + // maintainer may hand out access to other repos. + ErrAccepterNoWrite = errors.New(GrantLabel + " was applied by a user without write access") + // ErrAccepterIsBot is returned when a bot or App account applied the + // label: the hive must not be able to grant itself access by moving a + // label on an issue it works. + ErrAccepterIsBot = errors.New(GrantLabel + " was applied by a bot account") +) + +// RequestIssue is the request as the caller fetched it. Repo is the +// authorized repo holding the issue ("owner/name"); the Accepted* fields +// describe the latest "labeled" event for GrantLabel. +type RequestIssue struct { + Repo string + Number int + Title string + Body string + Closed bool + Labels []string + // AcceptedBy is the login that applied GrantLabel, empty when unknown. + AcceptedBy string + // AcceptedByPermission is that login's permission on Repo as GitHub + // reports it: "admin", "maintain", "write", "triage", "read" or "none". + AcceptedByPermission string + // AcceptedByIsBot reports that AcceptedBy is a bot or GitHub App account. + AcceptedByIsBot bool +} + +// Ref is the issue's "owner/repo#N" provenance string, the value every +// derived entry carries in DerivedFrom. +func (r RequestIssue) Ref() string { return fmt.Sprintf("%s#%d", r.Repo, r.Number) } + +// Options tune a derivation. +type Options struct { + // Host is the forge host URLs must be on to count as repositories; + // empty means DefaultHost. + Host string + // Authorized lists the project's own repos as qualified "owner/name" + // slugs. A repo that is already authorized can never also be a + // reference repo, so it is skipped with a reason. + Authorized []string +} + +// Skipped records a repository the issue named that did not become an entry +// and why, so a kick or the Settings editor can explain the difference. +type Skipped struct { + Repo string + Reason string +} + +// Skip reasons. +const ( + ReasonOutOfScope = "named under an out-of-scope field" + ReasonAuthorized = "already an authorized repo of this project" + ReasonOwnRepo = "is the repo holding the request issue" +) + +// Result is a successful derivation: the entries to merge and what was +// left out. Repos is empty when the issue names nothing in scope. +type Result struct { + Repos []config.ReferenceRepo + Skipped []Skipped +} + +// FormField is one rendered issue-form field: the heading text and the +// value below it, trimmed, with GitHub's "_No response_" read as empty. +type FormField struct { + Heading string + Value string +} + +// ParseIssueForm splits a GitHub issue-form body into its fields. A form +// renders each field as a "### <label>" line followed by the value until +// the next heading; text before the first heading is dropped. Headings +// repeated in the body are kept as separate fields in order. +func ParseIssueForm(body string) []FormField { + var ( + out []FormField + current *FormField + value []string + ) + flush := func() { + if current == nil { + return + } + current.Value = strings.TrimSpace(strings.Join(value, "\n")) + if current.Value == noResponse { + current.Value = "" + } + out = append(out, *current) + current, value = nil, nil + } + for _, line := range strings.Split(strings.ReplaceAll(body, "\r\n", "\n"), "\n") { + if heading, ok := strings.CutPrefix(line, headingPrefix); ok { + flush() + current = &FormField{Heading: strings.TrimSpace(heading)} + continue + } + if current != nil { + value = append(value, line) + } + } + flush() + return out +} + +// repoURL matches a repository URL on a host; group 1 is the owner, group 2 +// the name (with any ".git" suffix or trailing punctuation still attached). +// The host is quoted into the pattern at call time. +const repoURLPattern = `(?i)\bhttps?://(?:www\.)?%s/([A-Za-z0-9][A-Za-z0-9-]*)/([A-Za-z0-9._-]+)` + +// bareSlug matches a whole trimmed line or list item that is exactly +// "owner/name", optionally wrapped in backticks. +var bareSlug = regexp.MustCompile("^`?([A-Za-z0-9][A-Za-z0-9-]*)/([A-Za-z0-9._-]+)`?$") + +// listMarkers are the characters a Markdown list item may start with. +const listMarkers = "-*+" + +// trailingPunctuation is what a URL pasted into prose may drag along. +const trailingPunctuation = ".,;:)]>'\"`" + +// gitSuffix is stripped from a cloned-URL style repository name. +const gitSuffix = ".git" + +// ReposInText returns the "owner/name" repositories text names, in first +// appearance order, case-insensitively deduplicated. A repository is either +// a URL on host anywhere in the text, or a bare "owner/name" that is the +// whole line or list item: a path such as `src/docs/` inside a sentence is +// never a repository, and nothing on another host is. +func ReposInText(text, host string) []string { + if strings.TrimSpace(host) == "" { + host = DefaultHost + } + urlRe, err := regexp.Compile(fmt.Sprintf(repoURLPattern, regexp.QuoteMeta(strings.TrimSuffix(host, "/")))) + if err != nil { + return nil + } + var ( + out []string + seen = map[string]bool{} + ) + add := func(owner, name string) { + name = strings.TrimRight(name, trailingPunctuation) + name = strings.TrimSuffix(name, gitSuffix) + if owner == "" || name == "" { + return + } + slug := owner + "/" + name + key := strings.ToLower(slug) + if seen[key] { + return + } + seen[key] = true + out = append(out, slug) + } + for _, line := range strings.Split(text, "\n") { + item := strings.TrimSpace(line) + if len(item) > 1 && strings.ContainsRune(listMarkers, rune(item[0])) && item[1] == ' ' { + item = strings.TrimSpace(item[1:]) + } + if m := bareSlug.FindStringSubmatch(item); m != nil && !strings.HasSuffix(m[2], ".") { + add(m[1], m[2]) + continue + } + for _, m := range urlRe.FindAllStringSubmatch(line, -1) { + add(m[1], m[2]) + } + } + return out +} + +// HasWriteAccess reports whether a GitHub repository permission level lets +// its holder push: "admin", "maintain" or "write". +func HasWriteAccess(permission string) bool { + return writePermissions[strings.ToLower(strings.TrimSpace(permission))] +} + +// Accepted reports whether the issue carries GrantLabel (case-insensitive). +func (r RequestIssue) Accepted() bool { + for _, l := range r.Labels { + if strings.EqualFold(strings.TrimSpace(l), GrantLabel) { + return true + } + } + return false +} + +// checkGrant is the gate every derivation passes before reading the body. +func checkGrant(issue RequestIssue, intake workflowmodel.Intake) error { + ref := issue.Ref() + switch { + case !intake.DerivesReferenceRepos(): + return fmt.Errorf("%s: %w", ref, ErrNoIntakeFields) + case issue.Closed: + return fmt.Errorf("%s: %w", ref, ErrClosed) + case !issue.Accepted(): + return fmt.Errorf("%s: %w", ref, ErrNotAccepted) + case strings.TrimSpace(issue.AcceptedBy) == "": + return fmt.Errorf("%s: %w", ref, ErrAccepterUnknown) + case issue.AcceptedByIsBot: + return fmt.Errorf("%s: %w (%s)", ref, ErrAccepterIsBot, issue.AcceptedBy) + case !HasWriteAccess(issue.AcceptedByPermission): + return fmt.Errorf("%s: %w (%s has %q)", ref, ErrAccepterNoWrite, issue.AcceptedBy, issue.AcceptedByPermission) + } + return nil +} + +// headingIn reports whether heading matches one of the declared field +// labels, ignoring case and surrounding whitespace. +func headingIn(heading string, fields []string) bool { + for _, f := range fields { + if strings.EqualFold(strings.TrimSpace(f), strings.TrimSpace(heading)) { + return true + } + } + return false +} + +// Derive turns an accepted request issue into reference-repo entries. It +// refuses with one of the Err* reasons when the issue is not a grant, and +// otherwise returns one write-tier entry per repository named under the +// model's referenceRepoFields that is not named under an outOfScopeFields +// heading, not the issue's own repo and not an authorized repo. The order +// is the order of first appearance in the body. +func Derive(issue RequestIssue, intake workflowmodel.Intake, opts Options) (Result, error) { + if err := checkGrant(issue, intake); err != nil { + return Result{}, err + } + host := opts.Host + if strings.TrimSpace(host) == "" { + host = DefaultHost + } + excluded := map[string]bool{strings.ToLower(strings.TrimSpace(issue.Repo)): true} + authorized := make(map[string]bool, len(opts.Authorized)) + for _, a := range opts.Authorized { + authorized[strings.ToLower(strings.TrimSpace(a))] = true + } + fields := ParseIssueForm(issue.Body) + for _, f := range fields { + if headingIn(f.Heading, intake.OutOfScopeFields) { + for _, repo := range ReposInText(f.Value, host) { + excluded[strings.ToLower(repo)] = true + } + } + } + var ( + res Result + seen = map[string]bool{} + ref = issue.Ref() + ) + for _, f := range fields { + if !headingIn(f.Heading, intake.ReferenceRepoFields) { + continue + } + for _, repo := range ReposInText(f.Value, host) { + key := strings.ToLower(repo) + if seen[key] { + continue + } + seen[key] = true + switch { + case key == strings.ToLower(strings.TrimSpace(issue.Repo)): + res.Skipped = append(res.Skipped, Skipped{Repo: repo, Reason: ReasonOwnRepo}) + case excluded[key]: + res.Skipped = append(res.Skipped, Skipped{Repo: repo, Reason: ReasonOutOfScope}) + case authorized[key]: + res.Skipped = append(res.Skipped, Skipped{Repo: repo, Reason: ReasonAuthorized}) + default: + res.Repos = append(res.Repos, config.ReferenceRepo{ + Repo: repo, + Write: append([]string(nil), DerivedWrite...), + DerivedFrom: ref, + }) + } + } + } + return res, nil +} diff --git a/src/pkg/refderive/derive_test.go b/src/pkg/refderive/derive_test.go new file mode 100644 index 0000000000..b6231f4e75 --- /dev/null +++ b/src/pkg/refderive/derive_test.go @@ -0,0 +1,294 @@ +package refderive + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// techdocsIntake is what the cncf-techdocs-assessment model declares: the +// rendered headings of assessment-request.yml. +var techdocsIntake = workflowmodel.Intake{ + ReferenceRepoFields: []string{"Website or documentation repository", "Other repositories or sites in scope"}, + OutOfScopeFields: []string{"Out of scope"}, +} + +// fixture395 is the trimmed body of cncf/techdocs#395. +func fixture395(t *testing.T) string { + t.Helper() + body, err := os.ReadFile(filepath.Join("testdata", "techdocs-395.md")) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +// accepted395 is cncf/techdocs#395 as a maintainer accepted it. +func accepted395(t *testing.T) RequestIssue { + return RequestIssue{ + Repo: "cncf/techdocs", + Number: 395, + Title: "[Assessment request]: Hive Commons (pilot of the AI-assisted assessment workflow)", + Body: fixture395(t), + Labels: []string{"Docs analysis", GrantLabel, "stage/020-claimed", "author/human"}, + AcceptedBy: "nate-double-u", + AcceptedByPermission: "maintain", + } +} + +func TestParseIssueForm(t *testing.T) { + body := "preamble that is not a field\n### Project name\n\nHive Commons\n\n### Empty one\n\n_No response_\n\n### Multi\r\n\r\n- a\r\n- b\r\n### Last\nvalue" + got := ParseIssueForm(body) + want := []FormField{ + {Heading: "Project name", Value: "Hive Commons"}, + {Heading: "Empty one", Value: ""}, + {Heading: "Multi", Value: "- a\n- b"}, + {Heading: "Last", Value: "value"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("fields:\n got %+v\nwant %+v", got, want) + } + if got := ParseIssueForm(""); len(got) != 0 { + t.Fatalf("empty body: %+v", got) + } +} + +func TestReposInText(t *testing.T) { + cases := []struct { + name, text, host string + want []string + }{ + {"plain url", "https://github.com/hivecommons/docs", "", []string{"hivecommons/docs"}}, + {"url with path and prose", "see https://github.com/hivecommons/hive/blob/v5/README.md for details", "", []string{"hivecommons/hive"}}, + {"git suffix and trailing punctuation", "clone https://github.com/a/b.git, then https://github.com/c/d).", "", []string{"a/b", "c/d"}}, + {"www and http", "http://www.github.com/a/b", "", []string{"a/b"}}, + {"foreign host is text", "https://gitlab.com/a/b and https://example.com/c/d", "", nil}, + {"custom host", "https://ghe.example.com/a/b https://github.com/c/d", "ghe.example.com", []string{"a/b"}}, + {"bare slug whole line", "hivecommons/docs", "", []string{"hivecommons/docs"}}, + {"bare slug list item in backticks", "- `hivecommons/docs`\n* other/repo\n+ third/one", "", []string{"hivecommons/docs", "other/repo", "third/one"}}, + {"bare path inside a sentence is not a repo", "in-repo docs live under `src/docs/` and src/pkg here", "", nil}, + {"bare path with trailing slash is not a repo", "src/docs/", "", nil}, + {"dedupe case-insensitive keeps first", "https://github.com/HiveCommons/Docs\nhivecommons/docs", "", []string{"HiveCommons/Docs"}}, + {"not a slug", "github.com\ndocs.hivecommons.dev\n- [x] checked", "", nil}, + {"url in a list item with trailing note", "- https://github.com/hivecommons/hive — `src/docs/` (reference docs)", "", []string{"hivecommons/hive"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ReposInText(tc.text, tc.host) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } +} + +func TestHasWriteAccess(t *testing.T) { + for perm, want := range map[string]bool{"admin": true, "Maintain": true, " write ": true, "triage": false, "read": false, "none": false, "": false} { + if got := HasWriteAccess(perm); got != want { + t.Errorf("HasWriteAccess(%q) = %v, want %v", perm, got, want) + } + } +} + +func TestDeriveTechdocs395(t *testing.T) { + res, err := Derive(accepted395(t), techdocsIntake, Options{Authorized: []string{"cncf/techdocs"}}) + if err != nil { + t.Fatalf("derive: %v", err) + } + wantRepos := []string{"hivecommons/docs", "hivecommons/hivecommons.github.io", "hivecommons/hive"} + if len(res.Repos) != len(wantRepos) { + t.Fatalf("want %d repos, got %+v", len(wantRepos), res.Repos) + } + for i, rr := range res.Repos { + if rr.Repo != wantRepos[i] { + t.Errorf("repo %d: %q, want %q", i, rr.Repo, wantRepos[i]) + } + if !rr.WritesIssues() || !rr.WritesPRs() || rr.WriteLabel() != "issues+prs" { + t.Errorf("%s: tier %q, want issues+prs", rr.Repo, rr.WriteLabel()) + } + if rr.DerivedFrom != "cncf/techdocs#395" || !rr.Derived() { + t.Errorf("%s: DerivedFrom %q", rr.Repo, rr.DerivedFrom) + } + if rr.WriteTokenFile != "" { + t.Errorf("%s: a derived entry must never carry write_token_file", rr.Repo) + } + } + if len(res.Skipped) != 0 { + t.Fatalf("nothing in #395 should be skipped, got %+v", res.Skipped) + } + // The derived list reads back through the same accessors a hand-written + // one does. + project := config.ProjectConfig{Org: "cncf", Repos: []string{"techdocs"}, ReferenceRepos: res.Repos} + tier, ok := project.ReferenceRepoTier("HiveCommons/Hive") + if !ok || !tier.Issues || !tier.PRs { + t.Fatalf("tier lookup: ok=%v tier=%v", ok, tier) + } + if _, ok := project.ReferenceRepoTier("cncf/techdocs"); ok { + t.Fatal("the authorized repo must never read as a reference repo") + } + for _, rr := range res.Repos { + if rr.Owner() != "hivecommons" || rr.Name() == "" || strings.ContainsAny(rr.Repo, " :@") { + t.Fatalf("%q is not an explicit owner/name slug", rr.Repo) + } + } +} + +func TestDeriveRefuses(t *testing.T) { + base := accepted395(t) + cases := []struct { + name string + mutate func(*RequestIssue) + intake workflowmodel.Intake + want error + }{ + {"no intake fields", func(*RequestIssue) {}, workflowmodel.Intake{OutOfScopeFields: []string{"Out of scope"}}, ErrNoIntakeFields}, + {"closed", func(r *RequestIssue) { r.Closed = true }, techdocsIntake, ErrClosed}, + {"label removed", func(r *RequestIssue) { r.Labels = []string{"Docs analysis"} }, techdocsIntake, ErrNotAccepted}, + {"applier unknown", func(r *RequestIssue) { r.AcceptedBy = " " }, techdocsIntake, ErrAccepterUnknown}, + {"applier is a bot", func(r *RequestIssue) { r.AcceptedBy = "hivecommons-hive[bot]"; r.AcceptedByIsBot = true }, techdocsIntake, ErrAccepterIsBot}, + {"applier has triage only", func(r *RequestIssue) { r.AcceptedByPermission = "triage" }, techdocsIntake, ErrAccepterNoWrite}, + {"applier permission unknown", func(r *RequestIssue) { r.AcceptedByPermission = "" }, techdocsIntake, ErrAccepterNoWrite}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + issue := base + issue.Labels = append([]string(nil), base.Labels...) + tc.mutate(&issue) + res, err := Derive(issue, tc.intake, Options{}) + if !errors.Is(err, tc.want) { + t.Fatalf("err %v, want %v", err, tc.want) + } + if !strings.Contains(err.Error(), "cncf/techdocs#395") { + t.Fatalf("refusal must name the issue: %v", err) + } + if len(res.Repos) != 0 { + t.Fatalf("a refusal must derive nothing, got %+v", res.Repos) + } + }) + } +} + +func TestDeriveSkipsAndDedupes(t *testing.T) { + body := strings.Join([]string{ + "### Website or documentation repository", + "", + "https://github.com/acme/docs", + "", + "### Other repositories or sites in scope", + "", + "- https://github.com/ACME/docs (again, different case)", + "- https://github.com/acme/site", + "- https://github.com/acme/legacy", + "- https://github.com/cncf/techdocs", + "- https://github.com/cncf/toc", + "", + "### Out of scope", + "", + "- https://github.com/acme/legacy", + "", + }, "\n") + issue := accepted395(t) + issue.Body = body + issue.Number = 7 + res, err := Derive(issue, techdocsIntake, Options{Authorized: []string{"cncf/techdocs", "CNCF/toc"}}) + if err != nil { + t.Fatalf("derive: %v", err) + } + gotRepos := make([]string, 0, len(res.Repos)) + for _, rr := range res.Repos { + gotRepos = append(gotRepos, rr.Repo) + if rr.DerivedFrom != "cncf/techdocs#7" { + t.Errorf("%s: DerivedFrom %q", rr.Repo, rr.DerivedFrom) + } + } + if want := []string{"acme/docs", "acme/site"}; !reflect.DeepEqual(gotRepos, want) { + t.Fatalf("repos %v, want %v", gotRepos, want) + } + wantSkipped := []Skipped{ + {Repo: "acme/legacy", Reason: ReasonOutOfScope}, + {Repo: "cncf/techdocs", Reason: ReasonOwnRepo}, + {Repo: "cncf/toc", Reason: ReasonAuthorized}, + } + if !reflect.DeepEqual(res.Skipped, wantSkipped) { + t.Fatalf("skipped:\n got %+v\nwant %+v", res.Skipped, wantSkipped) + } +} + +func TestDeriveNothingNamedIsNotARefusal(t *testing.T) { + issue := accepted395(t) + issue.Body = "### Website or documentation repository\n\n_No response_\n\n### Other repositories or sites in scope\n\nOur docs live on https://docs.example.com only.\n" + res, err := Derive(issue, techdocsIntake, Options{}) + if err != nil { + t.Fatalf("derive: %v", err) + } + if len(res.Repos) != 0 || len(res.Skipped) != 0 { + t.Fatalf("want empty result, got %+v", res) + } +} + +func TestDeriveMatchesHeadingsCaseInsensitively(t *testing.T) { + issue := accepted395(t) + issue.Body = "### website or Documentation Repository \n\nhttps://github.com/acme/docs\n" + res, err := Derive(issue, techdocsIntake, Options{Host: "GitHub.com"}) + if err != nil { + t.Fatalf("derive: %v", err) + } + if len(res.Repos) != 1 || res.Repos[0].Repo != "acme/docs" { + t.Fatalf("got %+v", res.Repos) + } +} + +func TestDerivedWriteIsNeverWiderThanIssuesAndPRs(t *testing.T) { + known := map[string]bool{config.ReferenceWriteIssues: true, config.ReferenceWritePRs: true} + for _, w := range DerivedWrite { + if !known[w] { + t.Fatalf("DerivedWrite names unknown write kind %q", w) + } + } + tier := (config.ReferenceRepo{Write: DerivedWrite}).Tier() + if !tier.Issues || !tier.PRs || tier.String() != "issues+prs" || len(DerivedWrite) != len(known) { + t.Fatalf("DerivedWrite = %v, tier %v", DerivedWrite, tier) + } +} + +// TestIntakeSchema pins the workflow-model half: the field decodes from the +// model's JSON, an empty intake is omitted so existing models keep their +// hash, and a model without the field derives nothing. +func TestIntakeSchema(t *testing.T) { + doc := `{"apiVersion":"workflow.hivecommons.dev/v1alpha1","kind":"WorkflowModel","metadata":{"name":"m"}, + "spec":{"scope":{"forge":"github.com","repository":"cncf/techdocs"},"initial":"s","states":[{"id":"s","label":"S","owner":"human"}], + "intake":{"referenceRepoFields":["Website or documentation repository"],"outOfScopeFields":["Out of scope"]}}}` + var m workflowmodel.Model + if err := json.Unmarshal([]byte(doc), &m); err != nil { + t.Fatal(err) + } + if !m.Spec.Intake.DerivesReferenceRepos() || len(m.Spec.Intake.OutOfScopeFields) != 1 { + t.Fatalf("intake: %+v", m.Spec.Intake) + } + if err := m.Validate(); err != nil { + t.Fatalf("a model with intake must validate: %v", err) + } + with := m.Hash() + m.Spec.Intake = workflowmodel.Intake{} + if m.Spec.Intake.DerivesReferenceRepos() { + t.Fatal("empty intake must derive nothing") + } + without := m.Hash() + if with == without { + t.Fatal("declaring intake fields must change the spec hash") + } + raw, err := json.Marshal(m.Spec) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "intake") { + t.Fatalf("empty intake must be omitted from the spec JSON so stored models keep their hash:\n%s", raw) + } +} diff --git a/src/pkg/refderive/testdata/techdocs-395.md b/src/pkg/refderive/testdata/techdocs-395.md new file mode 100644 index 0000000000..80c16f16ff --- /dev/null +++ b/src/pkg/refderive/testdata/techdocs-395.md @@ -0,0 +1,52 @@ +### Project name + +Hive Commons (Hive) + +### Current CNCF maturity level + +Sandbox + +### Is this project applying to move levels? + +No + +### Project website + +https://hivecommons.dev + +### Documentation site + +https://docs.hivecommons.dev + +### Website or documentation repository + +https://github.com/hivecommons/docs + +### Other repositories or sites in scope + +- https://github.com/hivecommons/hivecommons.github.io (source for hivecommons.dev) +- https://github.com/hivecommons/hive — `src/docs/` (in-repo reference docs: api-reference, design, env-vars, agent-configuration) + +### Out of scope + +- Sibling project sites (Crux, hotshot, pluk, rationguard, promptargs, dibs, Spektacular) except where docs.hivecommons.dev links into them +- Source code and tests outside `src/docs/` + +### Documentation domains + +docs.hivecommons.dev +hivecommons.dev +github.com + +### Project contacts + +- clubanderson (Andrew Anderson, maintainer) — primary +- Hive Commons Discord #techdocs + +### Anything else + +Pilot request for the AI-assisted assessment workflow. Assessment deliverables (findings, implementation PRs) should land as PRs in `hivecommons/docs` (primary) and `hivecommons/hivecommons.github.io`; the analysis itself follows the normal `analyses/` layout here. + +### AI involvement disclosure + +- [x] I understand this assessment will be AI-drafted and human-reviewed, and that the project contacts named above will be notified before drafting begins. diff --git a/src/pkg/workflowmodel/model.go b/src/pkg/workflowmodel/model.go index 4d30b96249..b845db2657 100644 --- a/src/pkg/workflowmodel/model.go +++ b/src/pkg/workflowmodel/model.go @@ -112,8 +112,29 @@ type Spec struct { Labels LabelSet `json:"labels,omitzero"` Pipelines []Pipeline `json:"pipelines,omitempty"` Coverage Coverage `json:"coverage"` + Intake Intake `json:"intake,omitzero"` } +// Intake declares how the model's request form is read beyond the state +// rules (hivecommons/hive#18139). A request issue created from an issue +// form renders each field as a "### <label>" heading followed by its +// value; the lists below name those headings, not the form's field ids. +// Omitted on every model that has no assessment-style intake, so the hash +// of an existing model does not change. +type Intake struct { + // ReferenceRepoFields are the headings whose values name repositories + // the hive may study and write to once the request is accepted; they + // become derived project.reference_repos entries. + ReferenceRepoFields []string `json:"referenceRepoFields,omitempty"` + // OutOfScopeFields are the headings whose repositories are excluded + // even when another field names them. + OutOfScopeFields []string `json:"outOfScopeFields,omitempty"` +} + +// DerivesReferenceRepos reports whether the model declares any field that +// yields reference repos; a model without them derives nothing. +func (i Intake) DerivesReferenceRepos() bool { return len(i.ReferenceRepoFields) > 0 } + // Scope is the repository identity the model applies to. Forge is the // forge instance (github.com, a GHES host, a GitLab host), never implied // from the owner/name alone. From eadf000dab8dcf2287332351ff25e3d2c11abec0 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 00:56:19 +0000 Subject: [PATCH 195/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.250.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.250.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 12 ++++++++++++ changelog.d/added-18060-selfservice-registry-keys.md | 1 - .../fixed-18120-permissions-watcher-chown-eperm.md | 1 - .../fixed-18141-stage-engine-discussion-labels.md | 1 - changelog.d/fixed-18151-knowledge-advisor-429.md | 1 - 5 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/added-18060-selfservice-registry-keys.md delete mode 100644 changelog.d/fixed-18120-permissions-watcher-chown-eperm.md delete mode 100644 changelog.d/fixed-18141-stage-engine-discussion-labels.md delete mode 100644 changelog.d/fixed-18151-knowledge-advisor-429.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 998ab5329a..5f458f34fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.250.0) + +### Added + +- Hub: self-service (`byo-`) hives now have their own heartbeat credential ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18142](https://github.com/hivecommons/hive/issues/18142)). The hub mints a random key per registered hive, stores only its SHA-256 hash in `/data/hub-selfservice.json`, and verifies a `byo-` heartbeat against that hash and nothing else: a master-derived key never authenticates a `byo-` ID, and a self-service key authenticates no other hive. Keys rotate with a 10-minute overlap so a redeploy does not show the hive offline, and each accepted heartbeat stamps the record for the coming expiry sweep. The registration API and UI follow in later PRs; nothing changes for hosted or operator-provisioned hives. + +### Fixed + +- The permissions watcher no longer attempts, or logs per path, a chown it can never perform. After the entrypoint's privilege drop the hive process runs as `dev` (uid 1001) without `CAP_CHOWN`, so an entry another uid owns, such as a sandbox kick workspace under `/data/agents/sandbox/<slug>-<nonce>/.claude/skills/...` written by the Job pod's uid, always fails with `operation not permitted`; a surge of kicks minted ~4096 such paths between two ticks and the per-path deduper warned once for each of them (4087 `chown failed` WARN lines in ten minutes) before saturating. The watcher now decides ownership is unrepairable before the syscall, leaves the entry's mode alone as not ours, and reports all such entries as one summary line per tick: WARN with the count, a sample path and a hint when the condition appears or grows, DEBUG while it holds unchanged, nothing once it clears. Entries the process owns are still chowned and chmodded exactly as before (#18120). +- The stage engine now treats `architecture discussion` as a discussion label beside `proposal` and `kind/design`, and reads an `[RFC]` / `RFC:` title opener (after any leading emoji) as a discussion signal when no discussion label is present, so an unclaimed RFC derives `stage/018-open-for-discussion` (reason `open for discussion: architecture discussion` or `open for discussion: rfc title`) instead of showing as idle agent work at 010. A live claim or covering PR still stages 020 / 030 / 040 as before (#18141). +- The model-ranked knowledge advisor no longer burns its deliver arm on 429s from the model endpoint: a 429 is retried once after the upstream's `Retry-After` (a jittered 2 to 4 s backoff when absent) inside the existing `timeout_s`, a decision that still ends throttled is recorded as the new `rate-limited` outcome (distinct from `error`, excluded from the retro arm comparison) and is not charged against the per-agent-hour or per-day budgets, at most one advisor call is in flight across the whole hive on top of the per-agent guard, and the first 429 in each five-minute window logs at WARN with the `Retry-After`, `x-ratelimit-*` headers and response body while the rest log at DEBUG (#18151). + ## 2026-11-14 (v5.249.0) ### Added diff --git a/changelog.d/added-18060-selfservice-registry-keys.md b/changelog.d/added-18060-selfservice-registry-keys.md deleted file mode 100644 index 82dd199f16..0000000000 --- a/changelog.d/added-18060-selfservice-registry-keys.md +++ /dev/null @@ -1 +0,0 @@ -- Hub: self-service (`byo-`) hives now have their own heartbeat credential ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18142](https://github.com/hivecommons/hive/issues/18142)). The hub mints a random key per registered hive, stores only its SHA-256 hash in `/data/hub-selfservice.json`, and verifies a `byo-` heartbeat against that hash and nothing else: a master-derived key never authenticates a `byo-` ID, and a self-service key authenticates no other hive. Keys rotate with a 10-minute overlap so a redeploy does not show the hive offline, and each accepted heartbeat stamps the record for the coming expiry sweep. The registration API and UI follow in later PRs; nothing changes for hosted or operator-provisioned hives. diff --git a/changelog.d/fixed-18120-permissions-watcher-chown-eperm.md b/changelog.d/fixed-18120-permissions-watcher-chown-eperm.md deleted file mode 100644 index 2569b441f4..0000000000 --- a/changelog.d/fixed-18120-permissions-watcher-chown-eperm.md +++ /dev/null @@ -1 +0,0 @@ -- The permissions watcher no longer attempts, or logs per path, a chown it can never perform. After the entrypoint's privilege drop the hive process runs as `dev` (uid 1001) without `CAP_CHOWN`, so an entry another uid owns, such as a sandbox kick workspace under `/data/agents/sandbox/<slug>-<nonce>/.claude/skills/...` written by the Job pod's uid, always fails with `operation not permitted`; a surge of kicks minted ~4096 such paths between two ticks and the per-path deduper warned once for each of them (4087 `chown failed` WARN lines in ten minutes) before saturating. The watcher now decides ownership is unrepairable before the syscall, leaves the entry's mode alone as not ours, and reports all such entries as one summary line per tick: WARN with the count, a sample path and a hint when the condition appears or grows, DEBUG while it holds unchanged, nothing once it clears. Entries the process owns are still chowned and chmodded exactly as before (#18120). diff --git a/changelog.d/fixed-18141-stage-engine-discussion-labels.md b/changelog.d/fixed-18141-stage-engine-discussion-labels.md deleted file mode 100644 index ecc61bff51..0000000000 --- a/changelog.d/fixed-18141-stage-engine-discussion-labels.md +++ /dev/null @@ -1 +0,0 @@ -- The stage engine now treats `architecture discussion` as a discussion label beside `proposal` and `kind/design`, and reads an `[RFC]` / `RFC:` title opener (after any leading emoji) as a discussion signal when no discussion label is present, so an unclaimed RFC derives `stage/018-open-for-discussion` (reason `open for discussion: architecture discussion` or `open for discussion: rfc title`) instead of showing as idle agent work at 010. A live claim or covering PR still stages 020 / 030 / 040 as before (#18141). diff --git a/changelog.d/fixed-18151-knowledge-advisor-429.md b/changelog.d/fixed-18151-knowledge-advisor-429.md deleted file mode 100644 index 82837471cf..0000000000 --- a/changelog.d/fixed-18151-knowledge-advisor-429.md +++ /dev/null @@ -1 +0,0 @@ -- The model-ranked knowledge advisor no longer burns its deliver arm on 429s from the model endpoint: a 429 is retried once after the upstream's `Retry-After` (a jittered 2 to 4 s backoff when absent) inside the existing `timeout_s`, a decision that still ends throttled is recorded as the new `rate-limited` outcome (distinct from `error`, excluded from the retro arm comparison) and is not charged against the per-agent-hour or per-day budgets, at most one advisor call is in flight across the whole hive on top of the per-agent guard, and the first 429 in each five-minute window logs at WARN with the `Retry-After`, `x-ratelimit-*` headers and response body while the rest log at DEBUG (#18151). From 051a8b14fbdb46c7a8774767ba64c64bc64fd500 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 01:25:11 +0000 Subject: [PATCH 196/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.251.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.251.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ .../added-18139-reference-repos-from-request-issue.md | 1 - changelog.d/added-18144-selfservice-registration-api.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-18139-reference-repos-from-request-issue.md delete mode 100644 changelog.d/added-18144-selfservice-registration-api.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f458f34fc..45591f974b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.251.0) + +### Added + +- Workflow models gain `spec.intake.referenceRepoFields` / `outOfScopeFields`, `config.ReferenceRepo` gains a never-serialized `DerivedFrom` provenance, and the new `pkg/refderive` parser derives write-tier reference repos from an accepted assessment request issue, refusing with a typed reason when the request is closed, not `triage/accepted`, or accepted by a bot or a user without write access (#18139, #18157). +- Hub: signed-in users can register a hive they run themselves ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18144](https://github.com/hivecommons/hive/issues/18144)). Behind `HIVE_HUB_SELFSERVICE_REGISTRATION` (default off), `POST /api/hub/selfservice/hives` builds the ID as `byo-<login>-<name>`, proves `permissions.admin` on every listed repo with the caller's own GitHub token (refusing the whole registration by repo name otherwise), enforces `HIVE_HUB_SELFSERVICE_MAX_PER_USER` (default 5), and returns the heartbeat key once with Docker Compose, Podman and Kubernetes snippets; the owner can list, rotate and revoke their entries, a hub admin can revoke any, and every decision writes an `audit:` line without key material. The dashboard form follows in a later PR. + ## 2026-11-14 (v5.250.0) ### Added diff --git a/changelog.d/added-18139-reference-repos-from-request-issue.md b/changelog.d/added-18139-reference-repos-from-request-issue.md deleted file mode 100644 index 4c29cedc99..0000000000 --- a/changelog.d/added-18139-reference-repos-from-request-issue.md +++ /dev/null @@ -1 +0,0 @@ -- Workflow models gain `spec.intake.referenceRepoFields` / `outOfScopeFields`, `config.ReferenceRepo` gains a never-serialized `DerivedFrom` provenance, and the new `pkg/refderive` parser derives write-tier reference repos from an accepted assessment request issue, refusing with a typed reason when the request is closed, not `triage/accepted`, or accepted by a bot or a user without write access (#18139, #18157). diff --git a/changelog.d/added-18144-selfservice-registration-api.md b/changelog.d/added-18144-selfservice-registration-api.md deleted file mode 100644 index d0a85fe274..0000000000 --- a/changelog.d/added-18144-selfservice-registration-api.md +++ /dev/null @@ -1 +0,0 @@ -- Hub: signed-in users can register a hive they run themselves ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18144](https://github.com/hivecommons/hive/issues/18144)). Behind `HIVE_HUB_SELFSERVICE_REGISTRATION` (default off), `POST /api/hub/selfservice/hives` builds the ID as `byo-<login>-<name>`, proves `permissions.admin` on every listed repo with the caller's own GitHub token (refusing the whole registration by repo name otherwise), enforces `HIVE_HUB_SELFSERVICE_MAX_PER_USER` (default 5), and returns the heartbeat key once with Docker Compose, Podman and Kubernetes snippets; the owner can list, rotate and revoke their entries, a hub admin can revoke any, and every decision writes an `audit:` line without key material. The dashboard form follows in a later PR. From 40a9f3d73def381e899973303957f2096ebea7ac Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 20:52:22 -0500 Subject: [PATCH 197/223] =?UTF-8?q?=E2=9C=A8=20hub:=20self-service=20hive?= =?UTF-8?q?=20expiry=20sweep=20and=20hub-assigned=20self-hosted=20type=20(?= =?UTF-8?q?#18167)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub: self-service hive expiry sweep and hub-assigned self-hosted type A background sweep (HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL, default 10m) revokes the key and removes the registry entry of every byo- registration that went longer than HIVE_HUB_SELFSERVICE_EXPIRY (default 24h) without an accepted heartbeat, counted from LastAcceptedHeartbeatAt or from CreatedAt while there has been none. Removal takes the same path as the owner DELETE (selfService.revoke + removeRegistryEntry) and writes audit: selfservice_hive_expired with id, owner, last_heartbeat. handleHeartbeat now types byo- registry entries as self-hosted from the ID prefix; a hive_type the payload claims is ignored for that lane. Hosted and operator-listed entries keep their existing rule. Fixes #18146 Part of #18060 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🌱 test: drop hosted- heartbeat case that needs provisioning state A hosted- heartbeat answers 403 on a bare test hub before the type is assigned; the table test covers that branch of hubAssignedHiveType. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...added-18146-selfservice-expiry-and-type.md | 1 + src/docs/audit-log.md | 1 + src/docs/env-vars.md | 3 +- src/pkg/hub/background_pollers.go | 3 + src/pkg/hub/selfservice_api.go | 2 +- src/pkg/hub/selfservice_expiry.go | 121 ++++++++ src/pkg/hub/selfservice_expiry_test.go | 265 ++++++++++++++++++ src/pkg/hub/selfservice_registry.go | 4 +- src/pkg/hub/server.go | 12 +- 9 files changed, 399 insertions(+), 13 deletions(-) create mode 100644 changelog.d/added-18146-selfservice-expiry-and-type.md create mode 100644 src/pkg/hub/selfservice_expiry.go create mode 100644 src/pkg/hub/selfservice_expiry_test.go diff --git a/changelog.d/added-18146-selfservice-expiry-and-type.md b/changelog.d/added-18146-selfservice-expiry-and-type.md new file mode 100644 index 0000000000..bc8cd7fba8 --- /dev/null +++ b/changelog.d/added-18146-selfservice-expiry-and-type.md @@ -0,0 +1 @@ +- Hub: self-service (`byo-`) hives now expire. A background sweep (every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL`, default 10m) revokes the key and removes the registry entry of any registration that went longer than `HIVE_HUB_SELFSERVICE_EXPIRY` (default 24h) without an accepted heartbeat, writing `audit: selfservice_hive_expired`. The hub also assigns every `byo-` registry entry the hive type `self-hosted` from its ID prefix, ignoring any `hive_type` the heartbeat claims, so the dashboard and `/api/hub/*` listings can badge them (#18146, part of #18060). diff --git a/src/docs/audit-log.md b/src/docs/audit-log.md index 202a8423d2..0c38ac2b37 100644 --- a/src/docs/audit-log.md +++ b/src/docs/audit-log.md @@ -206,6 +206,7 @@ a key hash, the master secret or the user's GitHub token. | `selfservice_hive_registered` | A registration passed every check and the key was minted | `user`, `hive`, `repos` (count), `org` | | `selfservice_hive_rotated` | The owner rotated the heartbeat key | `user`, `hive`, `overlap` (how long the previous key keeps authenticating) | | `selfservice_hive_revoked` | The owner or a hub admin revoked the hive; the registry entry is removed in the same request and `registry entry removed` follows | `user`, `hive`, `owner`, `by_admin` | +| `selfservice_hive_expired` | The expiry sweep (`pkg/hub/selfservice_expiry.go`, every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL`) removed a hive that went longer than `HIVE_HUB_SELFSERVICE_EXPIRY` without an accepted heartbeat; the key is revoked and `registry entry removed` (with `by=selfservice-expiry-sweep`) follows | `id`, `owner`, `last_heartbeat` (RFC 3339, or `never` when no heartbeat was ever accepted), `expiry` | | `selfservice_hive_refused` | A registration was turned away and nothing was stored | `user`, `hive`, `reason` (`invalid_name`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `per_user_cap` with `max_per_user`, `no_github_token`, `no_github_login`, `not_repo_admin` with `repo`, `ownership_check_error` with `repo`) | ## Rotation and retention diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 4b4be0815d..818f67a0ed 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -86,7 +86,8 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_HUB_SECRET` | Required for spokes registered to a protected hub; optional for a standalone hub with `/data/saas/hub-secret.key` | `/data/saas/hub-secret.key` on the hub when present; no fallback for spoke heartbeat auth | Bearer secret for spoke heartbeats and hub/spoke SaaS APIs. | | `HIVE_HUB_SELFSERVICE_REGISTRATION` | No | off | Enables the self-service registration API (`/api/hub/selfservice/*`) that lets a signed-in user register a hive they run themselves under the reserved `byo-<login>-<name>` ID namespace, after proving `permissions.admin` on every listed repo with their own GitHub token ([#18060](https://github.com/hivecommons/hive/issues/18060)). Accepts `1/true/yes/on`. When off, every route but `GET /api/hub/selfservice/status` answers `404` and the status reports `enabled: false`. Read per request; no restart needed. | | `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | No | `5` | Maximum self-service hives one user may have registered at once; the next registration answers `429` until one is removed. | -| `HIVE_HUB_SELFSERVICE_EXPIRY` | No | `24h` | How long a self-service hive may go without an accepted heartbeat before the expiry sweep removes it. Reported by `GET /api/hub/selfservice/status` as `expiry`; the sweep itself lands with the follow-up to [#18060](https://github.com/hivecommons/hive/issues/18060). Go duration syntax. | +| `HIVE_HUB_SELFSERVICE_EXPIRY` | No | `24h` | How long a self-service hive may go without an accepted heartbeat before the expiry sweep revokes its key and removes its registry entry (counted from the last accepted heartbeat, or from registration while there has been none). Reported by `GET /api/hub/selfservice/status` as `expiry`. Go duration syntax. | +| `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` | No | `10m` | How often the self-service expiry sweep (`src/pkg/hub/selfservice_expiry.go`) runs. Each removal writes `audit: selfservice_hive_expired` (see [audit-log.md](audit-log.md)). Read at startup; restart after changing. Go duration syntax. | | `HIVE_HUB_AGENT_RESTART_PROBLEM_THRESHOLD` | No | `5` | Hosted hub threshold for surfacing agent restart storms on `/fleet`. When an agent reports at least this many restarts in the last 24 hours (or since the hub-side reset marker), the hive gets a restart problem chip and drift signal. | | `HIVE_COVERAGE_BADGE_URL` | No | none (the flagship hivecommons/hive gist for org `hivecommons`/`kubestellar` only) | Where the ci-maintainer card's coverage percentage comes from. Either an `http(s)` URL to a shields-style JSON badge (`{"message":"85%"}`) or an SVG badge, or `repo://<ref>/<path>` to read the badge file from the hive's primary repo through the GitHub App client — the form to use for a private repo, e.g. `repo://badges/coverage.svg` for octocov's default layout. Unset on a non-flagship hive, the card shows 0. | | `HIVE_BRANDING_CSS` | No | `<data>/branding/custom.css`, where `<data>` is the parent directory of the configured `data.agents_dir`, falling back to `/data` when that config field is empty — so the shipped default is `/data/branding/custom.css` | Absolute path of the operator override stylesheet the dashboard serves at `/branding/custom.css`. The index document links that URL unconditionally, so a missing file is the normal case and simply 404s. Read **per request**, so an edit takes effect on reload without a restart. The read is guarded (#5854): the file must be a regular file owned by the hive uid or root, not group- or world-writable, at most 128 KiB, and a symlink must resolve inside its own directory — anything else is refused with a logged warning, because whoever can write this path injects CSS into the operator's dashboard. See [Branding a hive, What the code enforces](branding.md#what-the-code-enforces). | diff --git a/src/pkg/hub/background_pollers.go b/src/pkg/hub/background_pollers.go index 32629120f9..0822db2313 100644 --- a/src/pkg/hub/background_pollers.go +++ b/src/pkg/hub/background_pollers.go @@ -53,6 +53,9 @@ func (s *HubServer) StartBackgroundPollers(ctx context.Context) <-chan struct{} start(s.StartBenchmarksPublisher) // Workflow marketplace nightly ingest + first-boot seed (#17620 phase 7). start(s.StartMarketplaceRefresh) + // Self-service (byo-) registrations past HIVE_HUB_SELFSERVICE_EXPIRY with + // no accepted heartbeat are revoked and dropped from the registry (#18146). + start(s.startSelfServiceExpirySweep) s.githubActivityMu.Lock() s.githubActivityCtx = ctx feed := s.githubActivityFeed diff --git a/src/pkg/hub/selfservice_api.go b/src/pkg/hub/selfservice_api.go index 899726ca8d..2d9ec28d5f 100644 --- a/src/pkg/hub/selfservice_api.go +++ b/src/pkg/hub/selfservice_api.go @@ -44,7 +44,7 @@ const ( selfServiceMaxPerUserEnv = "HIVE_HUB_SELFSERVICE_MAX_PER_USER" selfServiceMaxPerUserDefault = 5 // selfServiceExpiryEnv is how long a registered hive may go without an - // accepted heartbeat before the expiry sweep (a later child of #18060) + // accepted heartbeat before the expiry sweep (selfservice_expiry.go) // removes it. Read here so the status route can tell the form the window. selfServiceExpiryEnv = "HIVE_HUB_SELFSERVICE_EXPIRY" selfServiceExpiryDefault = 24 * time.Hour diff --git a/src/pkg/hub/selfservice_expiry.go b/src/pkg/hub/selfservice_expiry.go new file mode 100644 index 0000000000..0fee1f2f18 --- /dev/null +++ b/src/pkg/hub/selfservice_expiry.go @@ -0,0 +1,121 @@ +package hub + +import ( + "context" + "strings" + "time" +) + +// Self-service (byo-) hive lifecycle: expiry sweep and the hub-assigned hive +// type (RFC #18060 section 5, child #18146). + +const ( + // SelfServiceHiveType is the hub-assigned RegistryEntry.HiveType for every + // byo- hive. The hub derives it from the ID prefix; a value the heartbeat + // body claims is never consulted for this lane, so a self-hoster cannot + // present their hive as "hosted" in Active Hives or the fleet views. + SelfServiceHiveType = "self-hosted" + + // selfServiceSweepIntervalEnv sets how often the expiry sweep runs. The + // window itself is HIVE_HUB_SELFSERVICE_EXPIRY (selfServiceExpiry). + selfServiceSweepIntervalEnv = "HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL" + selfServiceSweepIntervalDefault = 10 * time.Minute + + // selfServiceExpiredBy is the actor recorded on the "registry entry + // removed" audit line when the sweep, not a user, removes an entry. + selfServiceExpiredBy = "selfservice-expiry-sweep" + + // selfServiceNeverHeartbeated is the last_heartbeat audit value for a + // registration that expired without one accepted heartbeat. + selfServiceNeverHeartbeated = "never" +) + +func selfServiceSweepInterval() time.Duration { + return durationEnv(selfServiceSweepIntervalEnv, selfServiceSweepIntervalDefault) +} + +// hubAssignedHiveType is the RegistryEntry.HiveType the heartbeat handler +// stores for hiveID. byo- hives always get SelfServiceHiveType regardless of +// what the payload claims; every other lane keeps the historical rule (a +// claimed type wins, hosted-/saas- IDs default to "hosted", the rest to +// "local"). +func hubAssignedHiveType(hiveID, claimed string) string { + if strings.HasPrefix(hiveID, SelfServiceHiveIDPrefix) { + return SelfServiceHiveType + } + if claimed != "" { + return sanitizeHeartbeatField(claimed) + } + if strings.HasPrefix(hiveID, "hosted-") || strings.HasPrefix(hiveID, "saas-") { + return "hosted" + } + return "local" +} + +// selfServiceLastSeen is the instant the expiry window counts from: the last +// accepted heartbeat, or the registration time while there has been none. +func selfServiceLastSeen(h SelfServiceHive) time.Time { + if !h.LastAcceptedHeartbeatAt.IsZero() { + return h.LastAcceptedHeartbeatAt + } + return h.CreatedAt +} + +// selfServiceExpired reports whether h has gone longer than window without an +// accepted heartbeat as of now. A record with no timestamps at all (a hand +// edited store file) is never expired by the sweep; it has no window to count. +func selfServiceExpired(h SelfServiceHive, now time.Time, window time.Duration) bool { + last := selfServiceLastSeen(h) + if last.IsZero() { + return false + } + return now.Sub(last) > window +} + +// sweepExpiredSelfServiceHives removes every registration past the expiry +// window through the same purge path as the owner DELETE: revoke the key, drop +// the registry entry (s.activity.forget and saveRegistryNow happen inside +// removeRegistryEntry), then one audit line per hive. It returns the removed +// IDs. Nothing here reads or logs key material. +func (s *HubServer) sweepExpiredSelfServiceHives() []string { + if s == nil || s.selfService == nil { + return nil + } + now := s.selfService.clock().UTC() + window := selfServiceExpiry() + var removed []string + for _, h := range s.selfService.list() { + if !selfServiceExpired(h, now, window) { + continue + } + if _, err := s.selfService.revoke(h.ID); err != nil { + s.logger.Error("self-service expiry: store write failed", "hive", h.ID, "error", err) + continue + } + s.removeRegistryEntry(h.ID, selfServiceExpiredBy) + lastHeartbeat := selfServiceNeverHeartbeated + if !h.LastAcceptedHeartbeatAt.IsZero() { + lastHeartbeat = h.LastAcceptedHeartbeatAt.UTC().Format(time.RFC3339) + } + s.logger.Info("audit: selfservice_hive_expired", "id", h.ID, "owner", h.Owner, "last_heartbeat", lastHeartbeat, "expiry", window.String()) + removed = append(removed, h.ID) + } + return removed +} + +// startSelfServiceExpirySweep runs sweepExpiredSelfServiceHives every +// selfServiceSweepInterval until ctx is cancelled. Started from +// StartBackgroundPollers beside the other hub pollers; a hub without a +// self-service store ticks harmlessly (the store's readers are nil-safe). +func (s *HubServer) startSelfServiceExpirySweep(ctx context.Context) { + ticker := time.NewTicker(selfServiceSweepInterval()) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.sweepExpiredSelfServiceHives() + } + } +} diff --git a/src/pkg/hub/selfservice_expiry_test.go b/src/pkg/hub/selfservice_expiry_test.go new file mode 100644 index 0000000000..0e71443efb --- /dev/null +++ b/src/pkg/hub/selfservice_expiry_test.go @@ -0,0 +1,265 @@ +package hub + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" +) + +// seedRegistryEntry puts a bare registry row for id in place, standing in for +// the row a first accepted heartbeat would have created. +func seedRegistryEntry(s *HubServer, id string) { + s.mu.Lock() + s.registry.Hives = append(s.registry.Hives, RegistryEntry{ID: id, Online: true, HiveType: SelfServiceHiveType}) + s.mu.Unlock() +} + +func TestSelfServiceExpirySweepRemovesPastExpiryAndKeepsWithin(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(selfServiceExpiryEnv, "") + s, logBuf := newSelfServiceHeartbeatHub(t) + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + s.selfService.now = func() time.Time { return now } + + const stale, fresh = "byo-alice-stale", "byo-alice-fresh" + staleKey, err := s.selfService.register(SelfServiceHive{ID: stale, Owner: "alice"}) + if err != nil { + t.Fatalf("register stale: %v", err) + } + freshKey, err := s.selfService.register(SelfServiceHive{ID: fresh, Owner: "alice"}) + if err != nil { + t.Fatalf("register fresh: %v", err) + } + staleBeat := now.Add(-selfServiceExpiryDefault - time.Minute) + if err := s.selfService.noteAcceptedHeartbeat(stale, staleBeat); err != nil { + t.Fatalf("stamp stale: %v", err) + } + if err := s.selfService.noteAcceptedHeartbeat(fresh, now.Add(-selfServiceExpiryDefault+time.Minute)); err != nil { + t.Fatalf("stamp fresh: %v", err) + } + seedRegistryEntry(s, stale) + seedRegistryEntry(s, fresh) + + removed := s.sweepExpiredSelfServiceHives() + if len(removed) != 1 || removed[0] != stale { + t.Fatalf("removed=%v, want [%s]", removed, stale) + } + if _, ok := s.selfService.get(stale); ok { + t.Fatal("expired hive still in the self-service store") + } + if s.selfServiceRegistryHas(stale) { + t.Fatal("expired hive still in the registry") + } + if _, ok := s.selfService.get(fresh); !ok { + t.Fatal("hive within the window was removed from the store") + } + if !s.selfServiceRegistryHas(fresh) { + t.Fatal("hive within the window was removed from the registry") + } + + // Snapshot the sweep's log output before the heartbeats below add their own + // lines. + logs := logBuf.String() + + // The revoked key is dead on the real heartbeat handler; the kept one is + // not. + if rec := postHeartbeatWithBearer(t, s, stale, staleKey); rec.Code != http.StatusUnauthorized { + t.Fatalf("expired key still authenticates: status=%d", rec.Code) + } + if rec := postHeartbeatWithBearer(t, s, fresh, freshKey); rec.Code != http.StatusOK { + t.Fatalf("kept key stopped authenticating: status=%d body=%s", rec.Code, rec.Body.String()) + } + + wantLine := "audit: selfservice_hive_expired" + if !strings.Contains(logs, wantLine) { + t.Fatalf("no %q audit line in logs:\n%s", wantLine, logs) + } + for _, kv := range []string{"id=" + stale, "owner=alice", "last_heartbeat=" + staleBeat.Format(time.RFC3339), "by=" + selfServiceExpiredBy} { + if !strings.Contains(logs, kv) { + t.Errorf("audit line missing %q:\n%s", kv, logs) + } + } + if strings.Contains(logs, "id="+fresh) { + t.Errorf("kept hive appeared in an audit line:\n%s", logs) + } + for _, secret := range []string{staleKey, freshKey, s.hubSecret} { + if strings.Contains(logBuf.String(), secret) { + t.Fatal("key material appeared in hub logs") + } + } + + // Idempotent: a second sweep finds nothing. + if again := s.sweepExpiredSelfServiceHives(); len(again) != 0 { + t.Fatalf("second sweep removed %v", again) + } +} + +func TestSelfServiceExpirySweepNeverHeartbeatedCountsFromCreatedAt(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(selfServiceExpiryEnv, "") + s, logBuf := newSelfServiceHeartbeatHub(t) + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + s.selfService.now = func() time.Time { return now } + + const id = "byo-bob-silent" + if _, err := s.selfService.register(SelfServiceHive{ID: id, Owner: "bob"}); err != nil { + t.Fatalf("register: %v", err) + } + if h, _ := s.selfService.get(id); !h.LastAcceptedHeartbeatAt.IsZero() || h.CreatedAt.IsZero() { + t.Fatalf("precondition: CreatedAt set, no heartbeat; got %+v", h) + } + + // Inside the window: kept. + now = now.Add(selfServiceExpiryDefault - time.Minute) + if removed := s.sweepExpiredSelfServiceHives(); len(removed) != 0 { + t.Fatalf("sweep inside the window removed %v", removed) + } + // Past it: gone, audited as never heartbeated. + now = now.Add(2 * time.Minute) + if removed := s.sweepExpiredSelfServiceHives(); len(removed) != 1 || removed[0] != id { + t.Fatalf("sweep past the window removed %v, want [%s]", removed, id) + } + if _, ok := s.selfService.get(id); ok { + t.Fatal("never-heartbeated hive survived the window") + } + if !strings.Contains(logBuf.String(), "last_heartbeat="+selfServiceNeverHeartbeated) { + t.Fatalf("audit line did not record last_heartbeat=%s:\n%s", selfServiceNeverHeartbeated, logBuf.String()) + } +} + +func TestSelfServiceExpiryWindowHonoursEnv(t *testing.T) { + t.Setenv(selfServiceExpiryEnv, "2h") + t.Setenv(selfServiceSweepIntervalEnv, "30s") + if got := selfServiceExpiry(); got != 2*time.Hour { + t.Fatalf("expiry=%s, want 2h", got) + } + if got := selfServiceSweepInterval(); got != 30*time.Second { + t.Fatalf("sweep interval=%s, want 30s", got) + } + t.Setenv(selfServiceSweepIntervalEnv, "not-a-duration") + if got := selfServiceSweepInterval(); got != selfServiceSweepIntervalDefault { + t.Fatalf("invalid interval fell back to %s, want %s", got, selfServiceSweepIntervalDefault) + } + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + h := SelfServiceHive{ID: "byo-x-y", CreatedAt: now.Add(-3 * time.Hour)} + if !selfServiceExpired(h, now, selfServiceExpiry()) { + t.Fatal("3h-old registration not expired by a 2h window") + } + if selfServiceExpired(SelfServiceHive{ID: "byo-x-z"}, now, selfServiceExpiry()) { + t.Fatal("a record with no timestamps must never expire") + } +} + +// TestSelfServiceExpirySweepLoopRunsOnTicker drives the real poller with a +// short interval and waits for the removal with testutil.Eventually; the clock +// the window is measured on is still the injected one. +func TestSelfServiceExpirySweepLoopRunsOnTicker(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(selfServiceExpiryEnv, "") + t.Setenv(selfServiceSweepIntervalEnv, "5ms") + s, _ := newSelfServiceHeartbeatHub(t) + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + s.selfService.now = func() time.Time { return now } + + const id = "byo-carol-lab" + if _, err := s.selfService.register(SelfServiceHive{ID: id, Owner: "carol"}); err != nil { + t.Fatalf("register: %v", err) + } + // Registration happened at `now`; the sweep measures against a clock + // already past the window, so the first tick removes it. + s.selfService.now = func() time.Time { return now.Add(selfServiceExpiryDefault + time.Minute) } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + s.startSelfServiceExpirySweep(ctx) + }() + testutil.Eventually(t, 5*time.Second, func() bool { + _, ok := s.selfService.get(id) + return !ok + }, "sweep loop did not remove the expired hive") + cancel() + <-done +} + +// postTypedHeartbeat sends a heartbeat whose body claims hive_type for hiveID +// and returns the HiveType the registry stored. +func postTypedHeartbeat(t *testing.T, s *HubServer, hiveID, bearer, claimedType string) (string, int) { + t.Helper() + body := `{"hive_id":"` + hiveID + `","org":"alice"` + if claimedType != "" { + body += `,"hive_type":"` + claimedType + `"` + } + body += `}` + req := httptest.NewRequest(http.MethodPost, "/api/heartbeat", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+bearer) + rec := httptest.NewRecorder() + s.handleHeartbeat(rec, req) + s.mu.RLock() + defer s.mu.RUnlock() + for _, h := range s.registry.Hives { + if h.ID == hiveID { + return h.HiveType, rec.Code + } + } + return "", rec.Code +} + +func TestSelfServiceHiveTypeIsHubAssigned(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + s, _ := newSelfServiceHeartbeatHub(t) + + const byo = "byo-alice-lab" + raw, err := s.selfService.register(SelfServiceHive{ID: byo, Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + // A byo- payload claiming to be hosted is still listed as self-hosted. + if got, code := postTypedHeartbeat(t, s, byo, raw, "hosted"); code != http.StatusOK || got != SelfServiceHiveType { + t.Fatalf("byo- claiming hosted: status=%d hiveType=%q, want %q", code, got, SelfServiceHiveType) + } + if got, code := postTypedHeartbeat(t, s, byo, raw, ""); code != http.StatusOK || got != SelfServiceHiveType { + t.Fatalf("byo- with no claim: status=%d hiveType=%q, want %q", code, got, SelfServiceHiveType) + } + + // The operator-listed lane is unchanged. (A hosted- heartbeat needs the + // provisioning state a bare test hub lacks and answers 403 before typing; + // TestHubAssignedHiveType covers that branch of the rule directly.) + const operator = "alice-lab" + if got, code := postTypedHeartbeat(t, s, operator, derivePerHiveKey(s.hubSecret, infoHeartbeatKey, operator), ""); code != http.StatusOK || got != "local" { + t.Fatalf("operator default: status=%d hiveType=%q, want local", code, got) + } + if got, code := postTypedHeartbeat(t, s, operator, derivePerHiveKey(s.hubSecret, infoHeartbeatKey, operator), "lite"); code != http.StatusOK || got != "lite" { + t.Fatalf("operator claimed type: status=%d hiveType=%q, want lite", code, got) + } +} + +func TestHubAssignedHiveType(t *testing.T) { + cases := []struct { + id, claimed, want string + }{ + {"byo-alice-lab", "", SelfServiceHiveType}, + {"byo-alice-lab", "hosted", SelfServiceHiveType}, + {"byo-alice-lab", "local", SelfServiceHiveType}, + {"hosted-x", "", "hosted"}, + {"saas-x", "", "hosted"}, + {"alice-lab", "", "local"}, + {"alice-lab", "lite", "lite"}, + } + for _, c := range cases { + if got := hubAssignedHiveType(c.id, c.claimed); got != c.want { + t.Errorf("hubAssignedHiveType(%q, %q)=%q, want %q", c.id, c.claimed, got, c.want) + } + } +} diff --git a/src/pkg/hub/selfservice_registry.go b/src/pkg/hub/selfservice_registry.go index 44bb2e62a2..ee749bd5a5 100644 --- a/src/pkg/hub/selfservice_registry.go +++ b/src/pkg/hub/selfservice_registry.go @@ -92,8 +92,8 @@ type SelfServiceHive struct { // permission on Repos. Zero when never checked. OwnershipCheckedAt time.Time `json:"ownership_checked_at,omitempty"` // LastAcceptedHeartbeatAt is the last heartbeat that passed the bearer - // check. Zero until the first one; the expiry sweep (a later child) - // removes registrations that never reach it. + // check. Zero until the first one; the expiry sweep (selfservice_expiry.go) + // counts the window from here, or from CreatedAt while it is zero. LastAcceptedHeartbeatAt time.Time `json:"last_accepted_heartbeat_at,omitempty"` } diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index bf9a127c89..79338bc5c1 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -2086,15 +2086,9 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { ContributorCount: clampInt(payload.Contributors.Registered, 0, 10_000), ActiveContributors: clampInt(payload.Contributors.Active, 0, 10_000), Owner: sanitizeHeartbeatField(payload.Owner), - HiveType: func() string { - if payload.HiveType != "" { - return sanitizeHeartbeatField(payload.HiveType) - } - if strings.HasPrefix(payload.HiveID, "hosted-") || strings.HasPrefix(payload.HiveID, "saas-") { - return "hosted" - } - return "local" - }(), + // byo- hives are typed by the hub from the ID prefix, never from the + // payload (#18146); see hubAssignedHiveType. + HiveType: hubAssignedHiveType(payload.HiveID, payload.HiveType), IsPublic: payload.IsPublic, LastHeartbeat: time.Now().UTC().Format(time.RFC3339), Health: payload.Health, From 436bf85992bf85579a3e04a1ef72fbbf3e50f9ac Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 21:02:44 -0500 Subject: [PATCH 198/223] =?UTF-8?q?=E2=9C=A8=20hub=20UI:=20Register=20your?= =?UTF-8?q?=20own=20hive=20form=20with=20one-time=20key,=20snippets,=20rot?= =?UTF-8?q?ate=20and=20remove=20(#18168)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub UI: Register your own hive form with one-time key, snippets, rotate and remove Behind HIVE_HUB_SELFSERVICE_REGISTRATION (GET /api/hub/selfservice/status on load). Both Register-your-own-hive CTAs open a modal when enabled and keep linking to /get-started#self-host otherwise. 201 shows the key once with Docker Compose, Podman and Kubernetes snippets; a Self-hosted hives section lists the owner's entries with Rotate key and Remove. Fixes #18145 Part of #18060 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 selfservice form: classes instead of style attributes, camelCase rows, test fixes Keeps the dashboard style ratchet at its baseline (no new style=, raw font sizes, rgba colours, px padding or radius), avoids the h.created_at hive-table guard by normalising list rows, and fixes two assertions in the new UX test. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...ded-18145-selfservice-registration-form.md | 1 + src/pkg/hub/assets/dashboard.html | 460 +++++++++++++++++- .../hub/saas_dashboard_selfservice_ux_test.go | 213 ++++++++ 3 files changed, 658 insertions(+), 16 deletions(-) create mode 100644 changelog.d/added-18145-selfservice-registration-form.md create mode 100644 src/pkg/hub/saas_dashboard_selfservice_ux_test.go diff --git a/changelog.d/added-18145-selfservice-registration-form.md b/changelog.d/added-18145-selfservice-registration-form.md new file mode 100644 index 0000000000..cf8849ec37 --- /dev/null +++ b/changelog.d/added-18145-selfservice-registration-form.md @@ -0,0 +1 @@ +- Hub dashboard: **Register your own hive** is now a form when the hub runs with `HIVE_HUB_SELFSERVICE_REGISTRATION` ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18145](https://github.com/hivecommons/hive/issues/18145)). Both CTAs (header and the "No hives yet" empty state) ask `GET /api/hub/selfservice/status` on load; with the flag off they keep linking to the self-host guide, with it on they open a modal that takes a name (shown with the fixed `byo-<login>-` prefix), org, one `owner/name` repository per line and an optional https dashboard URL, and posts to `/api/hub/selfservice/hives`. On success the heartbeat key is shown once with a copy button and a "copy before closing" warning, beside the hub URL and Docker Compose, Podman and Kubernetes snippets in tabs; the key lives only in the open dialog and is wiped from the DOM on close. Validation, not-an-admin (naming the refused repo), duplicate, per-user cap and GitHub-unreachable answers appear inline. A new **Self-hosted hives** section lists the signed-in user's own registrations with an online badge and last accepted heartbeat, **Rotate key** (confirm, then the new key once with the overlap note) and **Remove** (confirm, then revoke). diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index c1a2da62b5..081f052137 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -584,6 +584,45 @@ .hive-toast.success { background: rgba(116,223,154,0.9); } .hive-toast.error { background: rgba(255,126,126,0.9); } .hive-toast.info { background: rgba(128,191,255,0.9); } + /* Self-service registration modal and list (#18145). Colour comes from the + shared tokens only, so the light and dark themes carry through. */ + .ss-hidden { display: none !important; } + .ss-section { display: none; margin-top: var(--sp-9); } + .ss-section.open { display: block; } + .ss-section-head { display: flex; align-items: baseline; gap: var(--sp-4); margin-bottom: var(--sp-6); } + .ss-section-title { font-size: var(--fs-xl); color: var(--hub-dashboard-accent); margin: var(--sp-0); } + .ss-modal { display: none; position: fixed; inset: 0; background: var(--overlay-scrim); z-index: 100; align-items: center; justify-content: center; } + .ss-modal.open { display: flex; } + .ss-panel { background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-lg); max-width: 640px; width: 90%; max-height: 90vh; display: flex; flex-direction: column; } + .ss-head { font-size: var(--fs-xl); padding: var(--sp-9) var(--sp-9) var(--sp-6); margin: var(--sp-0); color: var(--hub-dashboard-accent); flex-shrink: 0; } + .ss-body { flex: 1; overflow-y: auto; padding: var(--sp-0) var(--sp-9); } + .ss-foot { display: flex; gap: var(--sp-5); justify-content: flex-end; padding: var(--sp-6) var(--sp-9); border-top: 1px solid var(--border); flex-shrink: 0; } + .ss-intro { font-size: var(--fs-base); color: var(--muted); margin-top: 0; } + .ss-input { width: 100%; padding: var(--sp-4) var(--sp-5); background: var(--bg); border: 1px solid var(--border); border-radius: var(--r); color: var(--text); font-size: var(--fs-base); font-family: inherit; box-sizing: border-box; } + .ss-label { display: block; font-size: var(--fs-sm); color: var(--muted); margin-bottom: var(--sp-2); } + .ss-sub { font-size: var(--fs-xs); } + .ss-prefix { color: var(--muted); font-size: var(--fs-base); white-space: nowrap; } + .ss-hint { font-size: var(--fs-xs); color: var(--muted); margin-top: var(--sp-2); } + .ss-warn, .ss-error { background: color-mix(in srgb, var(--status-error) 14%, transparent); border: 1px solid color-mix(in srgb, var(--status-error) 50%, transparent); border-radius: var(--r); padding: var(--sp-4) var(--sp-5); font-size: var(--fs-base); margin-bottom: var(--sp-5); color: var(--text); } + .ss-error { white-space: pre-wrap; overflow-wrap: anywhere; } + .ss-note { color: var(--muted); font-size: var(--fs-sm); } + .ss-cap { margin-bottom: var(--sp-5); } + .ss-field { margin-bottom: var(--sp-5); } + .ss-row { display: flex; gap: var(--sp-3); align-items: center; } + .ss-row-end { justify-content: flex-end; margin-bottom: var(--sp-2); } + .ss-code { flex: 1; min-width: 0; overflow-wrap: anywhere; background: var(--bg); border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-3) var(--sp-4); font-size: var(--fs-sm); color: var(--text); } + .ss-key { user-select: all; } + .ss-tabs { display: flex; gap: var(--sp-2); border-bottom: 1px solid var(--border); margin: var(--sp-6) 0 var(--sp-4); } + .ss-tab { background: transparent; border: 0; border-bottom: 2px solid transparent; color: var(--muted); padding: var(--sp-3) var(--sp-4); font-size: var(--fs-base); cursor: pointer; font-family: inherit; } + .ss-tab.active { color: var(--text); border-bottom-color: var(--hub-dashboard-accent); } + .ss-pre { background: var(--bg); border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-4); font-size: var(--fs-sm); overflow-x: auto; margin: 0; max-height: 260px; color: var(--text); } + .ss-hive { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3) var(--sp-5); padding: var(--sp-5); border: 1px solid var(--border); border-radius: var(--r); background: var(--surface); margin-bottom: var(--sp-4); } + .ss-hive-main { display: flex; align-items: center; gap: var(--sp-3); flex: 1 1 auto; min-width: 0; overflow-wrap: anywhere; } + .ss-hive-main a { font-size: var(--fs-sm); } + .ss-hive-meta { flex: 1 1 100%; color: var(--muted); font-size: var(--fs-sm); overflow-wrap: anywhere; } + .ss-hive-actions { display: flex; gap: var(--sp-3); margin-left: auto; } + .ss-badge { font-size: var(--fs-xs); padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); border: 1px solid var(--border); color: var(--muted); } + .ss-badge.on { color: var(--green); border-color: var(--green); } @keyframes spin { to { transform: rotate(360deg); } } @keyframes toast-in { from { transform: translateX(100%); opacity: 0; } to { transform: translateX(0); opacity: 1; } } .hive-confirm-overlay { position: fixed; inset: 0; background: rgba(0,0,0,0.6); z-index: 150; display: flex; align-items: center; justify-content: center; } @@ -800,17 +839,20 @@ <h1>My Hives</h1> <div style="display:flex;gap:var(--sp-4);align-items:center"> <button class="hv-btn btn-primary btn-primary" id="btn-send-banner-top" style="display:none" onclick="_bannerTargetHive=null;document.getElementById('banner-modal').style.display='flex';hiveHubRememberModalOpen(document.getElementById('banner-modal'));loadBannerHiveList()">Send Banner</button> <!-- Register-your-own-hive: for a user who self-installed a standalone - hive and wants to attach it to THIS hub. Points at the self-host - guide (get-started.html #self-host). - - NOTE: attaching a self-hosted hive is NOT self-service. Setting - HIVE_HUB_URL alone does not register anything — handleHeartbeat - rejects every unauthenticated beat with 401 whenever the hub has a - HIVE_HUB_SECRET configured (deliberate, #1077: "prevents registry - pollution from unauthenticated sources"). The operator must issue - the spoke a secret. The guide this links to now says so; keep the - two in step if either changes. --> - <a class="hv-btn btn-primary btn-primary" id="btn-register-hive" href="/get-started#self-host" style="background:var(--surface);color:var(--text);border:1px solid var(--border);text-decoration:none" title="You self-host the hive and attach it to this hub (requires a HIVE_HUB_SECRET issued by the hub operator)">Register your own hive</a> + hive and wants to attach it to THIS hub. + + Two behaviours, chosen by GET /api/hub/selfservice/status (#18060, + #18145). When the hub runs with HIVE_HUB_SELFSERVICE_REGISTRATION + the click opens the selfservice-modal form below: the hub proves + the user is a GitHub admin of the listed repos and issues the + spoke its own rotatable heartbeat key (shown once). When the flag + is off (the default) the anchor is a plain link to the self-host + guide (get-started.html #self-host), where the operator still has + to issue the spoke a secret: handleHeartbeat rejects every + unauthenticated beat with 401 whenever the hub has a + HIVE_HUB_SECRET configured (deliberate, #1077). The empty-state + copy of this CTA inside renderHives shares selfServiceCtaClick. --> + <a class="hv-btn btn-primary btn-primary" id="btn-register-hive" href="/get-started#self-host" onclick="return selfServiceCtaClick(event)" style="background:var(--surface);color:var(--text);border:1px solid var(--border);text-decoration:none" title="You self-host the hive and attach it to this hub">Register your own hive</a> <!-- Request-a-hive: routes to the EXISTING Request-a-Hive wizard at /get-started (files a provision request via POST /api/saas/request-provision). Shown when the user has NO hosted @@ -876,6 +918,18 @@ <h3 style="font-size:var(--fs-lg);color:var(--hub-dashboard-accent);margin-botto </div> </div> + <!-- Self-hosted hives the signed-in user registered through the + self-service form (#18145): online badge, last accepted heartbeat, + Rotate key and Remove. Hidden unless /api/hub/selfservice/status says + the feature is on; filled by renderSelfServiceList(). --> + <div id="selfservice-section" class="ss-section"> + <div class="ss-section-head"> + <h2 class="ss-section-title">Self-hosted hives</h2> + <span id="selfservice-cap" class="ss-note"></span> + </div> + <div id="selfservice-list"></div> + </div> + <div id="public-hives-section" style="display:none;margin-top:48px"> <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin-bottom:var(--sp-6)">Public Hives</h2> <div id="public-hives-container"></div> @@ -8885,11 +8939,13 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) /* Request-a-hive → the EXISTING Request-a-Hive wizard at /get-started (POST /api/saas/request-provision). We host it for you. */ '<a class="hv-btn btn-primary btn-primary" href="/get-started" style="text-decoration:none" title="We host the hive for you">Request a hive</a>' + - /* Register-your-own-hive → the EXISTING self-host guide at - /get-started#self-host. Not self-service: the operator must issue - the spoke a HIVE_HUB_SECRET or its heartbeats 401 (see #1077 and - the note on the other copy of this CTA above). */ - '<a class="hv-btn btn-primary btn-primary" href="/get-started#self-host" style="background:var(--surface);color:var(--text);border:1px solid var(--border);text-decoration:none" title="You self-host the hive and attach it to this hub">Register your own hive</a>' + + /* Register-your-own-hive → with HIVE_HUB_SELFSERVICE_REGISTRATION on, + selfServiceCtaClick opens the registration form (#18145); with the + flag off it falls through to the self-host guide at + /get-started#self-host, where the operator must still issue the + spoke a HIVE_HUB_SECRET or its heartbeats 401 (#1077; see the note + on the header copy of this CTA). */ + '<a class="hv-btn btn-primary btn-primary" href="/get-started#self-host" onclick="return selfServiceCtaClick(event)" style="background:var(--surface);color:var(--text);border:1px solid var(--border);text-decoration:none" title="You self-host the hive and attach it to this hub">Register your own hive</a>' + '</div>' + '<p style="color:var(--muted);font-size:0.85rem">…or contribute to a public hive below.</p>' + '</div>'; @@ -12178,6 +12234,327 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) } catch (e) { /* non-fatal */ } } + /* ---- Self-service (byo-) hive registration (#18060, #18145) ---- + Everything here is gated by GET /api/hub/selfservice/status. When the + hub runs without HIVE_HUB_SELFSERVICE_REGISTRATION the two "Register + your own hive" CTAs stay plain links to /get-started#self-host and none + of the fetches below happen. The raw heartbeat key exists only in the + 201 / rotate response and in the DOM of the open modal: it is never kept + in a variable that outlives the dialog, never written to browser + storage and never logged. Show/hide below is done with the ss-hidden / + open classes rather than style attributes (style ratchet). */ + var SELFSERVICE_ID_PREFIX = 'byo-'; + var SELFSERVICE_SNIPPET_TABS = [ + { key: 'docker_compose', label: 'Docker Compose' }, + { key: 'podman', label: 'Podman' }, + { key: 'kubernetes', label: 'Kubernetes' } + ]; + var SELFSERVICE_REPO_RE = /^[^\/\s]+\/[^\/\s]+$/; + var _selfService = { enabled: false, maxPerUser: 0, count: 0, expiry: '' }; + var _selfServiceHives = []; + + async function loadSelfServiceStatus() { + try { + var resp = await fetch('/api/hub/selfservice/status'); + if (!resp.ok) return; + var data = await resp.json(); + _selfService.enabled = !!data.enabled; + _selfService.maxPerUser = Number(data.max_per_user) || 0; + _selfService.count = Number(data.count) || 0; + _selfService.expiry = String(data.expiry || ''); + } catch (e) { return; } + if (!_selfService.enabled) return; + var btn = document.getElementById('btn-register-hive'); + if (btn) btn.title = 'Register a hive you run yourself: this hub issues it a heartbeat key'; + loadSelfServiceHives(); + } + /* Shared by the header CTA and the empty-state copy. Returning true lets + the anchor navigate to the guide (flag off); with the flag on the click + is swallowed and the form opens instead, so the page never navigates. */ + function selfServiceCtaClick(e) { + if (!_selfService.enabled) return true; + if (e && e.preventDefault) e.preventDefault(); + openSelfServiceModal(); + return false; + } + function selfServiceModal() { return document.getElementById('selfservice-modal'); } + function openSelfServiceModal() { + var m = selfServiceModal(); + if (!m) return; + selfServiceShowForm(); + var prefix = document.getElementById('ss-prefix'); + if (prefix) prefix.textContent = SELFSERVICE_ID_PREFIX + (_currentUser || 'you') + '-'; + var cap = document.getElementById('ss-cap'); + if (cap) cap.textContent = _selfService.maxPerUser ? (_selfService.count + ' of ' + _selfService.maxPerUser + ' self-hosted hives registered.') : ''; + m.classList.add('open'); + hiveHubRememberModalOpen(m); + var name = document.getElementById('ss-name'); + if (name) name.focus(); + } + function closeSelfServiceModal() { + var m = selfServiceModal(); + if (!m) return; + m.classList.remove('open'); + selfServiceClearKeyPanel(); + selfServiceSetError(''); + } + function selfServiceShow(el, on) { + if (el) el.classList.toggle('ss-hidden', !on); + } + /* The key panel is the only place the raw key lives; wipe it whenever it + is hidden, so a later DOM inspection or a reopened dialog has nothing. */ + function selfServiceClearKeyPanel() { + var k = document.getElementById('ss-key-panel'); + if (!k) return; + k.innerHTML = ''; + selfServiceShow(k, false); + } + function selfServiceShowForm() { + selfServiceShow(document.getElementById('ss-form-panel'), true); + selfServiceClearKeyPanel(); + selfServiceSetError(''); + ['ss-name', 'ss-org', 'ss-repos', 'ss-dash'].forEach(function(id) { + var el = document.getElementById(id); + if (el) el.value = ''; + }); + var title = document.getElementById('ss-title'); + if (title) title.textContent = 'Register your own hive'; + var close = document.getElementById('ss-close'); + if (close) close.textContent = 'Cancel'; + selfServiceShow(document.getElementById('ss-submit'), true); + selfServiceSetBusy(false); + } + function selfServiceSetError(msg) { + var el = document.getElementById('ss-error'); + if (!el) return; + el.textContent = msg || ''; + selfServiceShow(el, !!msg); + } + function selfServiceSetBusy(busy) { + var btn = document.getElementById('ss-submit'); + if (!btn) return; + btn.disabled = !!busy; + btn.textContent = busy ? 'Registering…' : 'Register'; + } + function selfServiceRepoLines(text) { + return String(text || '').split(/[\n,]+/).map(function(s) { return s.trim(); }).filter(Boolean); + } + async function submitSelfService() { + var name = (document.getElementById('ss-name').value || '').trim(); + var org = (document.getElementById('ss-org').value || '').trim(); + var repos = selfServiceRepoLines(document.getElementById('ss-repos').value); + var dash = (document.getElementById('ss-dash').value || '').trim(); + if (!name) { selfServiceSetError('Name is required.'); return; } + if (!repos.length) { selfServiceSetError('List at least one repository as owner/name, one per line.'); return; } + var bad = repos.filter(function(r) { return !SELFSERVICE_REPO_RE.test(r); }); + if (bad.length) { selfServiceSetError('Repository "' + bad[0] + '" must be owner/name.'); return; } + if (dash && !/^https:\/\//i.test(dash)) { selfServiceSetError('Dashboard URL must start with https://'); return; } + selfServiceSetError(''); + selfServiceSetBusy(true); + var resp, data = null; + try { + resp = await fetch('/api/hub/selfservice/hives', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: name, org: org, repos: repos, dashboard_url: dash }) + }); + try { data = await resp.json(); } catch (e) { data = null; } + } catch (e) { + selfServiceSetBusy(false); + selfServiceSetError('Network error: the hub did not answer. Try again.'); + return; + } + selfServiceSetBusy(false); + if (resp.status === 201 && data && data.key) { + _selfService.count++; + renderSelfServiceKey({ id: data.hive && data.hive.id, key: data.key, hubUrl: data.hub_url, snippets: data.snippets || {}, mode: 'registered' }); + loadSelfServiceHives(); + return; + } + selfServiceSetError(selfServiceErrorMessage(resp.status, data)); + } + /* Maps the API's status codes to one inline sentence. The server's own + error text is preferred when present (the 403 names the refused repo); + the fallbacks cover an empty or non-JSON body. */ + function selfServiceErrorMessage(status, data) { + var server = data && data.error ? String(data.error) : ''; + switch (status) { + case 400: return server || 'The hub rejected the form; check the fields and try again.'; + case 403: + if (data && data.repo) return 'Not registered: ' + (server || ('your GitHub account is not an admin of ' + data.repo)) + '. Only a repository admin can register a hive for it.'; + return server || 'Not allowed: sign out and back in with GitHub, then retry.'; + case 404: return server || 'Self-service registration is turned off on this hub.'; + case 409: return server || 'A hive with this name is already registered; pick another name.'; + case 429: return server || ('You already have the maximum of ' + _selfService.maxPerUser + ' self-hosted hives; remove one first.'); + case 502: return server || 'GitHub could not be reached to verify your repository permission; try again in a moment.'; + default: return server || ('Request failed (HTTP ' + status + ').'); + } + } + /* Renders the one-time key panel from a 201 or a rotate response. Every + value is server-provided and goes through esc/escAttr. */ + function renderSelfServiceKey(r) { + var form = document.getElementById('ss-form-panel'); + var panel = document.getElementById('ss-key-panel'); + if (!form || !panel) return; + selfServiceShow(form, false); + var title = document.getElementById('ss-title'); + if (title) title.textContent = r.mode === 'rotated' ? 'New key issued' : 'Hive registered'; + var close = document.getElementById('ss-close'); + if (close) close.textContent = 'Close'; + selfServiceShow(document.getElementById('ss-submit'), false); + var html = '<div class="ss-warn" role="alert"><strong>Copy the key before closing this dialog.</strong> It is shown once and never again: the hub stores only a hash. If you lose it, rotate it from the Self-hosted hives list.</div>'; + if (r.mode === 'rotated' && r.overlap) { + html += '<p class="ss-note">The previous key keeps working for ' + esc(r.overlap) + ' so a running spoke can be updated without a gap; after that only the new key is accepted.</p>'; + } + html += selfServiceValueRow('Hive ID', 'ss-out-id', r.id || '', 'hive ID'); + html += selfServiceValueRow('Heartbeat key (HIVE_HEARTBEAT_KEY)', 'ss-out-key', r.key || '', 'key', 'ss-key'); + html += selfServiceValueRow('Hub URL (HIVE_HUB_URL)', 'ss-out-hub', r.hubUrl || '', 'hub URL'); + html += '<div class="ss-tabs" role="tablist" aria-label="Deploy snippets">'; + SELFSERVICE_SNIPPET_TABS.forEach(function(t, i) { + html += '<button type="button" role="tab" class="ss-tab' + (i === 0 ? ' active' : '') + '" data-tab="' + t.key + '" aria-selected="' + (i === 0 ? 'true' : 'false') + '" onclick="selfServiceTab(\'' + t.key + '\')">' + esc(t.label) + '</button>'; + }); + html += '</div>'; + SELFSERVICE_SNIPPET_TABS.forEach(function(t, i) { + var text = r.snippets && r.snippets[t.key] ? String(r.snippets[t.key]) : ''; + html += '<div class="ss-snippet' + (i === 0 ? '' : ' ss-hidden') + '" id="ss-snippet-' + t.key + '" role="tabpanel">' + + '<div class="ss-row ss-row-end"><button type="button" class="hv-btn btn-primary btn-sm" onclick="selfServiceCopy(\'ss-snippet-text-' + t.key + '\',\'' + escAttr(t.label) + ' snippet\')">Copy</button></div>' + + '<pre class="ss-pre"><code id="ss-snippet-text-' + t.key + '">' + esc(text) + '</code></pre></div>'; + }); + html += '<p class="ss-note">Start the spoke with one of these; it shows as online in Self-hosted hives after its first accepted heartbeat. A registration that never heartbeats expires after ' + esc(_selfService.expiry || '24h') + '.</p>'; + panel.innerHTML = html; + selfServiceShow(panel, true); + } + function selfServiceValueRow(label, id, value, copyLabel, extraClass) { + return '<div class="ss-field"><label class="ss-label">' + esc(label) + '</label><div class="ss-row">' + + '<code class="ss-code' + (extraClass ? ' ' + extraClass : '') + '" id="' + id + '">' + esc(value) + '</code>' + + '<button type="button" class="hv-btn btn-primary btn-sm" onclick="selfServiceCopy(\'' + id + '\',\'' + escAttr(copyLabel) + '\')">Copy</button></div></div>'; + } + function selfServiceCopy(id, label) { + var el = document.getElementById(id); + if (!el) return; + copyHiveText(el.textContent, label); + } + function selfServiceTab(key) { + SELFSERVICE_SNIPPET_TABS.forEach(function(t) { + selfServiceShow(document.getElementById('ss-snippet-' + t.key), t.key === key); + }); + document.querySelectorAll('#ss-key-panel .ss-tab').forEach(function(b) { + var on = b.getAttribute('data-tab') === key; + b.classList.toggle('active', on); + b.setAttribute('aria-selected', on ? 'true' : 'false'); + }); + } + + async function loadSelfServiceHives() { + if (!_selfService.enabled) return; + var section = document.getElementById('selfservice-section'); + if (!section) return; + try { + var resp = await fetch('/api/hub/selfservice/hives'); + if (resp.status === 404) { _selfService.enabled = false; section.classList.remove('open'); return; } + if (!resp.ok) return; + var data = await resp.json(); + /* Normalised to the dashboard's camelCase row shape so the hive-table + guards against reading snake_case timestamps hold here too. */ + _selfServiceHives = ((data && data.hives) || []).map(function(v) { + return { + id: String(v.id || ''), + org: v.org || '', + repos: v.repos || [], + dashboardUrl: v.dashboard_url || '', + createdAt: v.created_at || '', + lastHeartbeatAt: v.last_accepted_heartbeat_at || '', + online: !!v.online + }; + }); + } catch (e) { return; } + _selfService.count = _selfServiceHives.length; + renderSelfServiceList(); + } + /* Go's zero time.Time serialises as year 0001, which Date.parse turns into + a large negative number; that and a missing field both read as never. */ + function selfServiceAgo(iso) { + if (!iso) return 'never'; + var t = Date.parse(iso); + if (!isFinite(t) || t <= 0) return 'never'; + var s = Math.max(0, Math.round((Date.now() - t) / 1000)); + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.round(s / 60) + 'm ago'; + if (s < 86400) return Math.round(s / 3600) + 'h ago'; + return Math.round(s / 86400) + 'd ago'; + } + function renderSelfServiceList() { + var section = document.getElementById('selfservice-section'); + var list = document.getElementById('selfservice-list'); + if (!section || !list) return; + section.classList.add('open'); + var cap = document.getElementById('selfservice-cap'); + if (cap) cap.textContent = _selfService.maxPerUser ? (_selfServiceHives.length + ' of ' + _selfService.maxPerUser) : ''; + if (!_selfServiceHives.length) { + list.innerHTML = '<p class="ss-note">No self-hosted hives yet. Use Register your own hive above to add one.</p>'; + return; + } + list.innerHTML = _selfServiceHives.map(function(h) { + var online = !!h.online; + return '<div class="ss-hive">' + + '<div class="ss-hive-main">' + + '<span class="online-dot ' + (online ? 'on' : 'off') + '" title="' + (online ? 'Online' : 'Offline') + '"></span>' + + '<strong>' + esc(h.id) + '</strong>' + + '<span class="ss-badge ' + (online ? 'on' : 'off') + '">' + (online ? 'online' : 'offline') + '</span>' + + (h.dashboardUrl ? '<a class="text-accent" href="' + escAttr(h.dashboardUrl) + '" target="_blank" rel="noopener">dashboard</a>' : '') + + '</div>' + + '<div class="ss-hive-actions">' + + '<button type="button" class="hv-btn btn-primary btn-sm" onclick="rotateSelfServiceHive(' + jsArg(h.id) + ')" title="Issue a new heartbeat key; the current one keeps working for a short overlap">Rotate key</button>' + + '<button type="button" class="hv-btn btn-danger btn-sm" onclick="removeSelfServiceHive(' + jsArg(h.id) + ')" title="Revoke the key and remove this hive from the hub">Remove</button>' + + '</div>' + + '<div class="ss-hive-meta">' + + (h.org ? 'org ' + esc(h.org) + ' · ' : '') + + 'repos ' + esc((h.repos || []).join(', ') || 'none') + ' · ' + + 'last heartbeat ' + esc(selfServiceAgo(h.lastHeartbeatAt)) + ' · ' + + 'registered ' + esc(selfServiceAgo(h.createdAt)) + + '</div></div>'; + }).join(''); + } + async function rotateSelfServiceHive(id) { + var ok = await hiveConfirm({ + title: 'Rotate key', + message: 'Issue a new heartbeat key for ' + id + '? The current key keeps working for a short overlap so you can update the spoke, then stops being accepted.', + confirmLabel: 'Rotate', + danger: false + }); + if (!ok) return; + var resp, data = null; + try { + resp = await fetch('/api/hub/selfservice/hives/' + encodeURIComponent(id) + '/rotate', { method: 'POST' }); + try { data = await resp.json(); } catch (e) { data = null; } + } catch (e) { hiveToast('Network error: key not rotated', 'error'); return; } + if (!resp.ok || !data || !data.key) { hiveToast(selfServiceErrorMessage(resp.status, data), 'error'); return; } + var m = selfServiceModal(); + if (m) { m.classList.add('open'); hiveHubRememberModalOpen(m); } + renderSelfServiceKey({ id: data.id || id, key: data.key, hubUrl: data.hub_url, overlap: data.overlap, snippets: data.snippets || {}, mode: 'rotated' }); + } + async function removeSelfServiceHive(id) { + var ok = await hiveConfirm({ + title: 'Remove self-hosted hive', + message: 'Remove ' + id + ' from this hub? Its key stops working at once and its heartbeats are rejected. The hive itself is not touched.', + confirmLabel: 'Remove' + }); + if (!ok) return; + var resp; + try { + resp = await fetch('/api/hub/selfservice/hives/' + encodeURIComponent(id), { method: 'DELETE' }); + } catch (e) { hiveToast('Network error: hive not removed', 'error'); return; } + if (!resp.ok) { + var data = null; + try { data = await resp.json(); } catch (e) { data = null; } + hiveToast(selfServiceErrorMessage(resp.status, data), 'error'); + return; + } + hiveToast('Removed ' + id, 'success'); + loadSelfServiceHives(); + loadHives(); + } + async function init() { /* Restore grouping, collapse state and saved views (including applying the default view) BEFORE the first loadHives(), so the initial render @@ -12200,6 +12577,9 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) scroll during a slow load is still captured. */ window.addEventListener('scroll', persistHiveScroll, {passive: true}); await loadUser(); + /* After loadUser so the byo-<login>- prefix is known; not awaited, the + CTAs keep their link behaviour until the status answers. */ + loadSelfServiceStatus(); await autoRequestAccessFromUrl(); await loadHives(); /* Restore AFTER the real rows are in the DOM — loadHives' renderHives is @@ -12218,6 +12598,9 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) init(); var POLL_INTERVAL_MS = 30000; setInterval(loadHives, POLL_INTERVAL_MS); + /* No-op while self-service registration is off; keeps the online badge + and last-heartbeat column of Self-hosted hives current otherwise. */ + setInterval(loadSelfServiceHives, POLL_INTERVAL_MS); /* Live-tick any claim-pending counters once a second so an assigned-but- unclaimed row's "claim pending · Nm" advances between the 30s polls. */ startAssignCounterTicker(); @@ -14646,6 +15029,51 @@ <h2 style="font-size:1.3rem;padding:var(--sp-9) var(--sp-9) var(--sp-6);margin:v </div> </div> + <!-- Self-service registration (#18060, #18145). Opened by the two + "Register your own hive" CTAs only when GET /api/hub/selfservice/status + answers enabled. The form panel is static markup; the key panel is + filled from the 201 / rotate response and emptied again by + closeSelfServiceModal(), so the raw key never outlives the open dialog. + Deliberately no backdrop-click or Escape close: dismissing the dialog by + accident would lose a key that is shown exactly once. --> + <div id="selfservice-modal" class="ss-modal"> + <div class="ss-panel" role="dialog" aria-modal="true" aria-labelledby="ss-title"> + <h2 id="ss-title" class="ss-head">Register your own hive</h2> + <div class="ss-body"> + <div id="ss-form-panel"> + <p class="ss-intro">You run the hive; this hub issues it a heartbeat key. You must be a GitHub admin of every repository you list: the hub checks that with the GitHub account you signed in with.</p> + <div class="ss-field"> + <label class="ss-label" for="ss-name">Name *</label> + <div class="ss-row"> + <code id="ss-prefix" class="ss-prefix">byo-</code> + <input id="ss-name" class="ss-input" type="text" placeholder="my-hive" autocomplete="off" maxlength="48"> + </div> + <div class="ss-hint">Letters, digits, '.', '_' and '-'. The hive ID is the prefix plus this name.</div> + </div> + <div class="ss-field"> + <label class="ss-label" for="ss-org">GitHub organization</label> + <input id="ss-org" class="ss-input" type="text" placeholder="my-org" autocomplete="off"> + </div> + <div class="ss-field"> + <label class="ss-label" for="ss-repos">Repositories * <span class="ss-sub">(one owner/name per line)</span></label> + <textarea id="ss-repos" class="ss-input" rows="3" placeholder="my-org/repo-one my-org/repo-two" spellcheck="false"></textarea> + </div> + <div class="ss-field"> + <label class="ss-label" for="ss-dash">Dashboard URL <span class="ss-sub">(optional, https)</span></label> + <input id="ss-dash" class="ss-input" type="url" placeholder="https://hive.example.com" autocomplete="off"> + </div> + <div id="ss-error" class="ss-error ss-hidden" role="alert"></div> + <div id="ss-cap" class="ss-note ss-cap"></div> + </div> + <div id="ss-key-panel" class="ss-hidden"></div> + </div> + <div class="ss-foot"> + <button class="hv-btn btn-primary" type="button" id="ss-close" onclick="closeSelfServiceModal()">Cancel</button> + <button class="hv-btn btn-primary btn-primary" type="button" id="ss-submit" onclick="submitSelfService()">Register</button> + </div> + </div> + </div> + <div id="access-modal" style="display:none;position:fixed;inset:0;background:rgba(0,0,0,0.6);z-index:100;align-items:center;justify-content:center"> <div class="access-modal-card"> <div class="access-modal-header"> diff --git a/src/pkg/hub/saas_dashboard_selfservice_ux_test.go b/src/pkg/hub/saas_dashboard_selfservice_ux_test.go new file mode 100644 index 0000000000..ac90968186 --- /dev/null +++ b/src/pkg/hub/saas_dashboard_selfservice_ux_test.go @@ -0,0 +1,213 @@ +package hub + +import ( + "regexp" + "strings" + "testing" +) + +// The self-service registration form (#18060, #18145) lives entirely inside +// the dashboardHTML raw string, so there is no Go function to exercise. These +// tests guard the wiring: the status call that gates the feature, both CTA +// hooks, the modal and its one-time key panel, the three snippet tabs, and +// the owner's list with Rotate and Remove. They also pin the invariants the +// RFC cares about: no key placeholder in the asset, the key panel is emptied +// on close, and the flag-off path still links to the self-host guide. + +// TestDashboardSelfServiceUXSymbols asserts every function the markup or the +// init path reaches for is defined and called. A typo is invisible to the Go +// compiler and only shows up as a dead click in the browser. +func TestDashboardSelfServiceUXSymbols(t *testing.T) { + html := dashScript(t) + for _, fn := range []string{ + "loadSelfServiceStatus", + "selfServiceCtaClick", + "openSelfServiceModal", + "closeSelfServiceModal", + "selfServiceClearKeyPanel", + "submitSelfService", + "selfServiceErrorMessage", + "renderSelfServiceKey", + "selfServiceCopy", + "selfServiceTab", + "loadSelfServiceHives", + "renderSelfServiceList", + "rotateSelfServiceHive", + "removeSelfServiceHive", + } { + if !strings.Contains(html, "function "+fn+"(") && + !strings.Contains(html, "async function "+fn+"(") { + t.Errorf("dashboardHTML is missing a definition for %s()", fn) + } + if strings.Count(html, fn+"(") < 2 { + t.Errorf("dashboardHTML defines %s() but never calls it", fn) + } + } +} + +// TestDashboardSelfServiceUXMarkup asserts the elements the JS reaches for by +// id are present: getElementById returning null is a silent no-op. +func TestDashboardSelfServiceUXMarkup(t *testing.T) { + html := dashScript(t) + for _, id := range []string{ + `id="btn-register-hive"`, + `id="selfservice-modal"`, + `id="ss-title"`, + `id="ss-form-panel"`, + `id="ss-prefix"`, + `id="ss-name"`, + `id="ss-org"`, + `id="ss-repos"`, + `id="ss-dash"`, + `id="ss-error"`, + `id="ss-cap"`, + `id="ss-key-panel"`, + `id="ss-close"`, + `id="ss-submit"`, + `id="selfservice-section"`, + `id="selfservice-list"`, + `id="selfservice-cap"`, + } { + if !strings.Contains(html, id) { + t.Errorf("dashboardHTML is missing element %s", id) + } + } +} + +// TestDashboardSelfServiceUXAPIWiring asserts the asset talks to exactly the +// routes selfservice_api.go registers, and that the status call is what gates +// the feature. +func TestDashboardSelfServiceUXAPIWiring(t *testing.T) { + html := dashScript(t) + for _, want := range []string{ + `fetch('/api/hub/selfservice/status')`, + `fetch('/api/hub/selfservice/hives')`, + `fetch('/api/hub/selfservice/hives', {`, + `method: 'POST'`, + `/rotate', { method: 'POST' }`, + `{ method: 'DELETE' }`, + `loadSelfServiceStatus();`, + } { + if !strings.Contains(html, want) { + t.Errorf("dashboardHTML is missing %q", want) + } + } + for _, status := range []string{"case 400:", "case 403:", "case 409:", "case 429:", "case 502:"} { + if !strings.Contains(html, status) { + t.Errorf("selfServiceErrorMessage does not map %s", status) + } + } + // The 403 body carries the refused repo; the message must name it. + if !strings.Contains(html, "data.repo") { + t.Error("the 403 message does not use the repo the API names") + } +} + +// TestDashboardSelfServiceUXCTAs asserts both "Register your own hive" CTAs +// keep their link to the self-host guide (the flag-off path) and share the +// click hook that opens the form when the flag is on, and that the comments no +// longer claim the operator must issue HIVE_HUB_SECRET as the only path. +func TestDashboardSelfServiceUXCTAs(t *testing.T) { + html := dashScript(t) + const hook = `href="/get-started#self-host" onclick="return selfServiceCtaClick(event)"` + if got := strings.Count(html, hook); got != 2 { + t.Errorf("expected both Register-your-own-hive CTAs to carry %q, found %d", hook, got) + } + if !strings.Contains(html, `if (!_selfService.enabled) return true;`) { + t.Error("selfServiceCtaClick must let the anchor navigate when the flag is off") + } + if strings.Contains(html, "requires a HIVE_HUB_SECRET issued by the hub operator") { + t.Error("the CTA tooltip still says the operator must issue HIVE_HUB_SECRET") + } + if strings.Contains(html, "attaching a self-hosted hive is NOT self-service") { + t.Error("the CTA comment still says attaching is not self-service; it should describe HIVE_HUB_SELFSERVICE_REGISTRATION") + } + if !strings.Contains(html, "HIVE_HUB_SELFSERVICE_REGISTRATION") { + t.Error("the CTA comment does not name the flag that switches the behaviour") + } +} + +// TestDashboardSelfServiceUXKeyPanel pins the one-time key contract: the key +// is rendered only from the response, the panel is emptied on close, the +// three snippet tabs exist, and nothing in the asset looks like a key. +func TestDashboardSelfServiceUXKeyPanel(t *testing.T) { + html := dashScript(t) + for _, want := range []string{ + `key: data.key`, + `'ss-out-key', r.key || ''`, + `k.innerHTML = '';`, + `selfServiceClearKeyPanel();`, + `shown once and never again`, + `{ key: 'docker_compose', label: 'Docker Compose' }`, + `{ key: 'podman', label: 'Podman' }`, + `{ key: 'kubernetes', label: 'Kubernetes' }`, + `r.snippets[t.key]`, + `esc(r.overlap)`, + } { + if !strings.Contains(html, want) { + t.Errorf("dashboardHTML is missing %q", want) + } + } + // closeSelfServiceModal must wipe the key panel, not just hide the modal. + closeIdx := strings.Index(html, "function closeSelfServiceModal()") + if closeIdx < 0 { + t.Fatal("closeSelfServiceModal is missing") + } + closeBody := html[closeIdx:] + if end := strings.Index(closeBody, "\n }\n"); end > 0 { + closeBody = closeBody[:end] + } + if !strings.Contains(closeBody, "selfServiceClearKeyPanel()") { + t.Error("closeSelfServiceModal does not clear the key panel") + } + // The key must never be persisted or logged by the self-service code. + ssStart := strings.Index(html, "Self-service (byo-) hive registration (#18060, #18145)") + ssEnd := strings.Index(html, "async function init()") + if ssStart < 0 || ssEnd < ssStart { + t.Fatal("cannot locate the self-service script block") + } + block := html[ssStart:ssEnd] + for _, forbidden := range []string{"localStorage", "sessionStorage", "console.log", "console.debug", "console.info"} { + if strings.Contains(block, forbidden) { + t.Errorf("self-service script block must not use %s", forbidden) + } + } + // Show/hide is done with classes so the dashboard style ratchet + // (pkg/dashboard/webstatic) does not grow; the script block and the modal + // markup must stay free of style attributes. + modalStart := strings.Index(html, `id="selfservice-modal"`) + modalEnd := strings.Index(html[modalStart:], `id="access-modal"`) + if modalStart < 0 || modalEnd < 0 { + t.Fatal("cannot locate the self-service modal markup") + } + for name, chunk := range map[string]string{"script block": block, "modal markup": html[modalStart : modalStart+modalEnd]} { + if strings.Contains(chunk, "style=") { + t.Errorf("self-service %s uses a style attribute; use the .ss-* classes", name) + } + } + // No hard-coded key placeholder anywhere in the asset: the raw key comes + // from the API response and nowhere else. + placeholder := regexp.MustCompile(`(?i)(HIVE_HEARTBEAT_KEY\s*[:=]\s*["']?[A-Za-z0-9+/_-]{16,}|byo-[a-z0-9-]+-[a-z0-9]+\.[A-Za-z0-9_-]{20,})`) + if m := placeholder.FindString(html); m != "" { + t.Errorf("dashboardHTML contains what looks like a hard-coded key: %q", m) + } +} + +// TestDashboardSelfServiceUXEscaping asserts every server-provided string in +// the list and key panel is escaped before it reaches innerHTML. +func TestDashboardSelfServiceUXEscaping(t *testing.T) { + html := dashScript(t) + for _, want := range []string{ + `esc(h.id)`, + `esc(h.org)`, + `esc((h.repos || []).join(', ')`, + `escAttr(h.dashboardUrl)`, + `jsArg(h.id)`, + `esc(value)`, + `esc(text)`, + } { + if !strings.Contains(html, want) { + t.Errorf("dashboardHTML is missing escaped render %q", want) + } + } +} From 644af5925e279a3bb6423b63c9e0da61b5322c99 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 21:10:51 -0500 Subject: [PATCH 199/223] =?UTF-8?q?=F0=9F=90=9B=20hub:=20harden=20self-ser?= =?UTF-8?q?vice=20registration=20(cap=20race,=20hyphen=20IDs,=20dot=20segm?= =?UTF-8?q?ents)=20(#18170)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Re-check HIVE_HUB_SELFSERVICE_MAX_PER_USER inside the store under the write lock (registerCapped); the handler's count runs before up to 20 GitHub calls and raced for concurrent POSTs from one user. - Build self-service IDs as byo-<login>--<name>. GitHub logins never contain "--" or end in "-", so the boundary is unambiguous; refuse logins that break that rule and names starting with "-". - isValidRepoRef refuses "." and ".." segments before url.JoinPath. - Tests: concurrent registers at cap-1 give exactly one 201; hyphen logins a-b/c and a/b-c get distinct IDs; owner/.. and owner/. are 400. Closes #18166 Hive-Run: hivecommons/hive#18166 Hive-Plan: #18060 self-service registration follow-ups Hive-Spec: hub-selfservice-registration#18166 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- ...8166-selfservice-registration-hardening.md | 1 + src/docs/api-reference/hub-server.md | 68 ++++---- src/docs/audit-log.md | 2 +- src/docs/env-vars.md | 2 +- src/pkg/hub/selfservice_api.go | 45 ++++- src/pkg/hub/selfservice_api_test.go | 163 ++++++++++++++++-- src/pkg/hub/selfservice_registry.go | 33 +++- src/pkg/hub/selfservice_registry_test.go | 41 +++++ src/pkg/hub/server.go | 7 +- 9 files changed, 300 insertions(+), 62 deletions(-) create mode 100644 changelog.d/security-18166-selfservice-registration-hardening.md diff --git a/changelog.d/security-18166-selfservice-registration-hardening.md b/changelog.d/security-18166-selfservice-registration-hardening.md new file mode 100644 index 0000000000..2f2fc212c1 --- /dev/null +++ b/changelog.d/security-18166-selfservice-registration-hardening.md @@ -0,0 +1 @@ +- **Self-service hive registration hardening** ([#18166](https://github.com/hivecommons/hive/issues/18166), part of [#18060](https://github.com/hivecommons/hive/issues/18060)): `HIVE_HUB_SELFSERVICE_MAX_PER_USER` is now enforced again under the self-service store lock, so concurrent `POST /api/hub/selfservice/hives` requests from one user can no longer all pass the pre-ownership count and exceed the cap (the late refusal answers `429` and logs `selfservice_hive_refused` with `reason=per_user_cap`). New self-service IDs use `byo-<login>--<name>` instead of `byo-<login>-<name>`, so logins that contain hyphens can no longer collide or squat each other's IDs (`a-b` + `c` and `a` + `b-c` both used to become `byo-a-b-c`); a login containing `--` or starting or ending with `-` is refused with `reason=invalid_login`, and a name may not start with `-`. Hives registered before this change keep their existing IDs and keys. Repo references `.` and `..` (for example `owner/..`) are now refused everywhere repo refs are validated, so they can no longer be resolved as path segments in the GitHub ownership-check URL. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 8666408551..cca609f0a4 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,47 +35,47 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks/<hash>/<YYYY-MM>.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1819` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1820` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1822` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1823` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1743` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1744` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1745` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1746` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1747` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1758` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1759` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1760` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1761` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1746` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1747` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1748` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1749` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1750` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1761` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1762` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1763` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1764` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=<id>[,<id>…]` (repeatable) and `?level=<n>` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence-<hive|fleet>-<from>-<to>.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#<n>`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1769` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1779` | -| `GET` | `/api/hub/selfservice/status` | Hub auth | Self-service registration status: `{enabled, max_per_user, count, expiry}`; always answers so the dashboard can decide whether **Register your own hive** opens the form or the self-hosting guide ([#18060](https://github.com/hivecommons/hive/issues/18060)) | `pkg/hub/server.go:1783` | -| `GET` | `/api/hub/selfservice/hives` | Hub auth | The caller's own self-service (`byo-`) hives: `{hives[]}` of `{id, owner, org, repos, dashboard_url, created_at, ownership_checked_at, last_accepted_heartbeat_at, online}`; never a key or a hash. `404` while `HIVE_HUB_SELFSERVICE_REGISTRATION` is off | `pkg/hub/server.go:1784` | -| `POST` | `/api/hub/selfservice/hives` | Hub auth | Register a self-hosted hive: body `{name, org, repos[], dashboard_url}`; the ID is `byo-<github login>-<name>`. Proves `permissions.admin` on every `owner/name` repo with the caller's own stored GitHub token (`403` naming the refused repo, nothing stored); `400` invalid or reserved name, bare-org repo, or non-`https`/private `dashboard_url`; `409` ID already registered; `429` over `HIVE_HUB_SELFSERVICE_MAX_PER_USER`. `201 {hive, key, hub_url, snippets{docker_compose, podman, kubernetes}}`: the raw key appears here once and only its hash is stored. `404` while the flag is off | `pkg/hub/server.go:1785` | -| `POST` | `/api/hub/selfservice/hives/{id}/rotate` | Hub auth | Owner only: mint a new heartbeat key for `{id}`; the previous key keeps authenticating for `overlap` (10 minutes). `200 {id, key, hub_url, overlap, snippets}`, raw key once. Another user's hive, a missing hive and a hub admin who does not own it all get `404` | `pkg/hub/server.go:1786` | -| `DELETE` | `/api/hub/selfservice/hives/{id}` | Hub auth | Owner or hub admin: revoke the key and remove the registry entry. `200 {removed, id}`; another user's hive is `404` | `pkg/hub/server.go:1787` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1788` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1789` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1790` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1791` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1792` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1793` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1794` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1800` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1801` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1808` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1809` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1810` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1827` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1832` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1834` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1772` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1782` | +| `GET` | `/api/hub/selfservice/status` | Hub auth | Self-service registration status: `{enabled, max_per_user, count, expiry}`; always answers so the dashboard can decide whether **Register your own hive** opens the form or the self-hosting guide ([#18060](https://github.com/hivecommons/hive/issues/18060)) | `pkg/hub/server.go:1786` | +| `GET` | `/api/hub/selfservice/hives` | Hub auth | The caller's own self-service (`byo-`) hives: `{hives[]}` of `{id, owner, org, repos, dashboard_url, created_at, ownership_checked_at, last_accepted_heartbeat_at, online}`; never a key or a hash. `404` while `HIVE_HUB_SELFSERVICE_REGISTRATION` is off | `pkg/hub/server.go:1787` | +| `POST` | `/api/hub/selfservice/hives` | Hub auth | Register a self-hosted hive: body `{name, org, repos[], dashboard_url}`; the ID is `byo-<github login>--<name>` (the double hyphen keeps IDs unambiguous, since logins contain hyphens but never `--`). Proves `permissions.admin` on every `owner/name` repo with the caller's own stored GitHub token (`403` naming the refused repo, nothing stored); `400` invalid or reserved name, a login that contains `--` or starts or ends with `-`, bare-org or `.`/`..` repo, or non-`https`/private `dashboard_url`; `409` ID already registered; `429` over `HIVE_HUB_SELFSERVICE_MAX_PER_USER` (enforced again under the store lock, so concurrent registrations cannot exceed it). `201 {hive, key, hub_url, snippets{docker_compose, podman, kubernetes}}`: the raw key appears here once and only its hash is stored. `404` while the flag is off | `pkg/hub/server.go:1788` | +| `POST` | `/api/hub/selfservice/hives/{id}/rotate` | Hub auth | Owner only: mint a new heartbeat key for `{id}`; the previous key keeps authenticating for `overlap` (10 minutes). `200 {id, key, hub_url, overlap, snippets}`, raw key once. Another user's hive, a missing hive and a hub admin who does not own it all get `404` | `pkg/hub/server.go:1789` | +| `DELETE` | `/api/hub/selfservice/hives/{id}` | Hub auth | Owner or hub admin: revoke the key and remove the registry entry. `200 {removed, id}`; another user's hive is `404` | `pkg/hub/server.go:1790` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1791` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1792` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1793` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1794` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1795` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1796` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1800` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1803` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1804` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1811` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1812` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1813` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1830` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1835` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1837` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/docs/audit-log.md b/src/docs/audit-log.md index 0c38ac2b37..90f78247b3 100644 --- a/src/docs/audit-log.md +++ b/src/docs/audit-log.md @@ -207,7 +207,7 @@ a key hash, the master secret or the user's GitHub token. | `selfservice_hive_rotated` | The owner rotated the heartbeat key | `user`, `hive`, `overlap` (how long the previous key keeps authenticating) | | `selfservice_hive_revoked` | The owner or a hub admin revoked the hive; the registry entry is removed in the same request and `registry entry removed` follows | `user`, `hive`, `owner`, `by_admin` | | `selfservice_hive_expired` | The expiry sweep (`pkg/hub/selfservice_expiry.go`, every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL`) removed a hive that went longer than `HIVE_HUB_SELFSERVICE_EXPIRY` without an accepted heartbeat; the key is revoked and `registry entry removed` (with `by=selfservice-expiry-sweep`) follows | `id`, `owner`, `last_heartbeat` (RFC 3339, or `never` when no heartbeat was ever accepted), `expiry` | -| `selfservice_hive_refused` | A registration was turned away and nothing was stored | `user`, `hive`, `reason` (`invalid_name`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `per_user_cap` with `max_per_user`, `no_github_token`, `no_github_login`, `not_repo_admin` with `repo`, `ownership_check_error` with `repo`) | +| `selfservice_hive_refused` | A registration was turned away and nothing was stored | `user`, `hive`, `reason` (`invalid_name`, `invalid_login`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `per_user_cap` with `max_per_user`, `no_github_token`, `no_github_login`, `not_repo_admin` with `repo`, `ownership_check_error` with `repo`) | ## Rotation and retention diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 818f67a0ed..64425ef5f7 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -84,7 +84,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_HUB_KIOSK_CTA_TEXT` | No | `🐝 Want this for your repos?` (`kioskcta.DefaultText`) | Copy line above the URL on that badge. | | `HIVE_HUB_LEGACY_COOKIE_DOMAIN` | No | none | Optional transition-only hub SSO cookie domain to expire alongside the active cookie domain while accepting any carried `hive_hub_user` cookie value during a host migration. Writes still target the domain derived from `HIVE_HUB_PUBLIC_URL`; unset disables the extra legacy-domain cleanup. | | `HIVE_HUB_SECRET` | Required for spokes registered to a protected hub; optional for a standalone hub with `/data/saas/hub-secret.key` | `/data/saas/hub-secret.key` on the hub when present; no fallback for spoke heartbeat auth | Bearer secret for spoke heartbeats and hub/spoke SaaS APIs. | -| `HIVE_HUB_SELFSERVICE_REGISTRATION` | No | off | Enables the self-service registration API (`/api/hub/selfservice/*`) that lets a signed-in user register a hive they run themselves under the reserved `byo-<login>-<name>` ID namespace, after proving `permissions.admin` on every listed repo with their own GitHub token ([#18060](https://github.com/hivecommons/hive/issues/18060)). Accepts `1/true/yes/on`. When off, every route but `GET /api/hub/selfservice/status` answers `404` and the status reports `enabled: false`. Read per request; no restart needed. | +| `HIVE_HUB_SELFSERVICE_REGISTRATION` | No | off | Enables the self-service registration API (`/api/hub/selfservice/*`) that lets a signed-in user register a hive they run themselves under the reserved `byo-<login>--<name>` ID namespace, after proving `permissions.admin` on every listed repo with their own GitHub token ([#18060](https://github.com/hivecommons/hive/issues/18060)). Accepts `1/true/yes/on`. When off, every route but `GET /api/hub/selfservice/status` answers `404` and the status reports `enabled: false`. Read per request; no restart needed. | | `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | No | `5` | Maximum self-service hives one user may have registered at once; the next registration answers `429` until one is removed. | | `HIVE_HUB_SELFSERVICE_EXPIRY` | No | `24h` | How long a self-service hive may go without an accepted heartbeat before the expiry sweep revokes its key and removes its registry entry (counted from the last accepted heartbeat, or from registration while there has been none). Reported by `GET /api/hub/selfservice/status` as `expiry`. Go duration syntax. | | `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` | No | `10m` | How often the self-service expiry sweep (`src/pkg/hub/selfservice_expiry.go`) runs. Each removal writes `audit: selfservice_hive_expired` (see [audit-log.md](audit-log.md)). Read at startup; restart after changing. Go duration syntax. | diff --git a/src/pkg/hub/selfservice_api.go b/src/pkg/hub/selfservice_api.go index 2d9ec28d5f..2cd965b754 100644 --- a/src/pkg/hub/selfservice_api.go +++ b/src/pkg/hub/selfservice_api.go @@ -16,7 +16,8 @@ package hub // // Invariants: // - Every ID this file produces starts with SelfServiceHiveIDPrefix and is -// built as byo-<github login>-<name>; nothing a caller sends can escape it. +// built as byo-<github login>--<name>; nothing a caller sends can escape it, +// and no two (login, name) pairs produce the same ID (#18166). // - The raw key is in the 201 and the rotate body and nowhere else: not in // the list route, not in a log line, not on disk (only its hash is). // - HIVE_HUB_SECRET is never consulted here and never appears in a response. @@ -64,18 +65,27 @@ const ( // selfServiceDashboardURLScheme is the only scheme a dashboard_url may use. selfServiceDashboardURLScheme = "https" + + // selfServiceIDSeparator joins the GitHub login and the hive name in a + // byo- ID. A single '-' was ambiguous because logins contain hyphens + // (login a-b + name c and login a + name b-c both gave byo-a-b-c, #18166). + // GitHub logins never contain "--" and never end in '-', so the first "--" + // after the prefix is always the boundary; selfServiceBuildID refuses a + // login that breaks either rule rather than mint an ambiguous ID. + selfServiceIDSeparator = "--" ) // selfServiceReservedNamePrefixes are hive ID namespaces a self-service name -// may not borrow. The final ID is always byo-<login>-<name>, so none of these +// may not borrow. The final ID is always byo-<login>--<name>, so none of these // could become the ID's prefix; refusing them in the NAME keeps a registration -// from reading like a hosted hive in the dashboard (byo-alice-hosted-prod). +// from reading like a hosted hive in the dashboard (byo-alice--hosted-prod). var selfServiceReservedNamePrefixes = []string{"hosted-", "saas-", SelfServiceHiveIDPrefix} // selfServiceRefusalReasons are the audit-log reason values for // selfservice_hive_refused. const ( selfServiceRefusedInvalidName = "invalid_name" + selfServiceRefusedInvalidLogin = "invalid_login" selfServiceRefusedReservedPrefix = "reserved_prefix" selfServiceRefusedDuplicateID = "duplicate_id" selfServiceRefusedInvalidRepos = "invalid_repos" @@ -369,10 +379,15 @@ func selfServiceValidateDashboardURL(ctx context.Context, raw string) (string, e return u.String(), nil } -// selfServiceBuildID derives byo-<login>-<name> from the GitHub login and the +// selfServiceBuildID derives byo-<login>--<name> from the GitHub login and the // caller's name, refusing names that cannot become a valid hive ID or that -// borrow a reserved namespace. +// borrow a reserved namespace, and logins that would make the ID ambiguous. func selfServiceBuildID(githubLogin, name string) (string, string, error) { + login := strings.ToLower(strings.TrimSpace(githubLogin)) + if login == "" || !isValidName(login) || strings.Contains(login, selfServiceIDSeparator) || + strings.HasPrefix(login, "-") || strings.HasSuffix(login, "-") { + return "", selfServiceRefusedInvalidLogin, errors.New("your GitHub login cannot form an unambiguous self-hosted hive ID") + } name = strings.ToLower(strings.TrimSpace(name)) if name == "" { return "", selfServiceRefusedInvalidName, errors.New("name is required") @@ -380,12 +395,17 @@ func selfServiceBuildID(githubLogin, name string) (string, string, error) { if strings.Contains(name, "/") || !isValidName(name) { return "", selfServiceRefusedInvalidName, errors.New("name may contain only letters, digits, '.', '_' and '-'") } + // A leading '-' would merge into the separator (a + -b gives byo-a---b, + // whose first "--" no longer marks the boundary). + if strings.HasPrefix(name, "-") { + return "", selfServiceRefusedInvalidName, errors.New("name may not start with '-'") + } for _, p := range selfServiceReservedNamePrefixes { if strings.HasPrefix(name, p) { return "", selfServiceRefusedReservedPrefix, fmt.Errorf("name may not start with the reserved prefix %q", p) } } - id := SelfServiceHiveIDPrefix + strings.ToLower(strings.TrimSpace(githubLogin)) + "-" + name + id := SelfServiceHiveIDPrefix + login + selfServiceIDSeparator + name if !isValidName(id) || !IsSelfServiceHiveID(id) { return "", selfServiceRefusedInvalidName, errors.New("name is too long for a hive ID") } @@ -491,15 +511,24 @@ func (s *HubServer) handleSelfServiceRegister(w http.ResponseWriter, r *http.Req } now := time.Now().UTC() - rawKey, err := s.selfService.register(SelfServiceHive{ + // The cap is enforced again inside the store, under the lock that inserts + // the record: the count above ran before up to selfServiceMaxRepos GitHub + // calls, so concurrent POSTs from one user could all have passed it. + maxPerUser := selfServiceMaxPerUser() + rawKey, err := s.selfService.registerCapped(SelfServiceHive{ ID: id, Owner: login, Org: org, Repos: repos, DashboardURL: dashboardURL, OwnershipCheckedAt: now, - }) + }, maxPerUser) if err != nil { if errors.Is(err, errSelfServiceExists) { s.auditSelfServiceRefused(login, id, selfServiceRefusedDuplicateID) writeJSONError(w, http.StatusConflict, "a hive with this ID is already registered") return } + if errors.Is(err, errSelfServiceCapReached) { + s.auditSelfServiceRefused(login, id, selfServiceRefusedCap, "max_per_user", maxPerUser) + writeJSONError(w, http.StatusTooManyRequests, fmt.Sprintf("you already have %d self-hosted hives registered; remove one first", maxPerUser)) + return + } s.logger.Error("self-service registration: store write failed", "user", login, "hive", id, "error", err) writeJSONError(w, http.StatusInternalServerError, "could not persist the registration") return diff --git a/src/pkg/hub/selfservice_api_test.go b/src/pkg/hub/selfservice_api_test.go index ad82866985..1884909a41 100644 --- a/src/pkg/hub/selfservice_api_test.go +++ b/src/pkg/hub/selfservice_api_test.go @@ -36,6 +36,9 @@ type selfServiceTestEnv struct { // stub owner answers admin=false; unknown repos answer 404. adminRepos map[string]bool now *time.Time + // repoCheckGate, when set, runs inside the GitHub stub before it answers, + // so a test can hold concurrent registrations inside the ownership proof. + repoCheckGate func() } type lockedBuffer struct { @@ -79,6 +82,9 @@ func newSelfServiceTestEnv(t *testing.T) *selfServiceTestEnv { w.WriteHeader(http.StatusUnauthorized) return } + if env.repoCheckGate != nil { + env.repoCheckGate() + } repo := strings.TrimPrefix(r.URL.Path, "/repos/") admin, known := env.adminRepos[repo] if !known { @@ -215,8 +221,8 @@ func TestSelfServiceAPIUnauthenticated(t *testing.T) { "status": env.status(t, ""), "list": env.list(t, ""), "register": env.register(t, "", selfServiceHappyBody), - "rotate": env.rotate(t, "", "byo-alice-lab"), - "delete": env.remove(t, "", "byo-alice-lab"), + "rotate": env.rotate(t, "", "byo-alice--lab"), + "delete": env.remove(t, "", "byo-alice--lab"), } for name, rec := range cases { if rec.Code != http.StatusUnauthorized { @@ -234,8 +240,8 @@ func TestSelfServiceAPIFlagOffIsNotFound(t *testing.T) { cases := map[string]*httptest.ResponseRecorder{ "list": env.list(t, selfServiceTestUser), "register": env.register(t, selfServiceTestUser, selfServiceHappyBody), - "rotate": env.rotate(t, selfServiceTestUser, "byo-alice-lab"), - "delete": env.remove(t, selfServiceTestUser, "byo-alice-lab"), + "rotate": env.rotate(t, selfServiceTestUser, "byo-alice--lab"), + "delete": env.remove(t, selfServiceTestUser, "byo-alice--lab"), } for name, rec := range cases { if rec.Code != http.StatusNotFound { @@ -276,8 +282,8 @@ func TestSelfServiceAPIRegisterHappyPath(t *testing.T) { if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if resp.Hive.ID != "byo-alice-lab" { - t.Fatalf("id = %q, want byo-alice-lab", resp.Hive.ID) + if resp.Hive.ID != "byo-alice--lab" { + t.Fatalf("id = %q, want byo-alice--lab", resp.Hive.ID) } if resp.Hive.Owner != selfServiceTestUser || resp.Hive.Org != "alice" || len(resp.Hive.Repos) != 2 || resp.Hive.DashboardURL != "https://lab.example.org/" { t.Fatalf("view = %+v", resp.Hive) @@ -295,17 +301,17 @@ func TestSelfServiceAPIRegisterHappyPath(t *testing.T) { // The store holds only the hash, and the hub accepts the raw key as this // hive's heartbeat bearer and no other hive's. - stored, ok := env.s.selfService.get("byo-alice-lab") + stored, ok := env.s.selfService.get("byo-alice--lab") if !ok { t.Fatal("hive not stored") } if stored.KeyHash != hashSelfServiceKey(resp.Key) || stored.KeyHash == resp.Key { t.Fatal("store does not hold exactly the hash of the returned key") } - if gen, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, "byo-alice-lab"); !ok || gen != selfServiceBearerGeneration { + if gen, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, "byo-alice--lab"); !ok || gen != selfServiceBearerGeneration { t.Fatalf("raw key does not authenticate the hive: gen=%d ok=%v", gen, ok) } - if _, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, "byo-alice-other"); ok { + if _, ok := env.s.verifyHeartbeatBearerGeneration(resp.Key, "byo-alice--other"); ok { t.Fatal("raw key authenticated a different hive") } if !strings.Contains(env.logs.String(), "audit: selfservice_hive_registered") { @@ -323,7 +329,7 @@ func TestSelfServiceAPIRegisterHappyPath(t *testing.T) { if err := json.Unmarshal(listRec.Body.Bytes(), &listed); err != nil { t.Fatal(err) } - if len(listed.Hives) != 1 || listed.Hives[0].ID != "byo-alice-lab" { + if len(listed.Hives) != 1 || listed.Hives[0].ID != "byo-alice--lab" { t.Fatalf("list = %+v", listed.Hives) } for _, forbidden := range []string{resp.Key, stored.KeyHash, "key_hash", grantableTestSecret} { @@ -341,7 +347,7 @@ func TestSelfServiceAPIRegisterHappyPath(t *testing.T) { func TestSelfServiceAPIRegisterRefusals(t *testing.T) { env := newSelfServiceTestEnv(t) // Pre-existing registry entry for the duplicate case. - env.s.registry.Hives = []RegistryEntry{{ID: "byo-alice-taken"}} + env.s.registry.Hives = []RegistryEntry{{ID: "byo-alice--taken"}} cases := []struct { name string body string @@ -358,6 +364,10 @@ func TestSelfServiceAPIRegisterRefusals(t *testing.T) { {"duplicate registry id", `{"name":"taken","repos":["alice/lab"]}`, http.StatusConflict, selfServiceRefusedDuplicateID}, {"no repos", `{"name":"lab","repos":[]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, {"bare org repo", `{"name":"lab","repos":["alice"]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, + {"dot-dot repo segment", `{"name":"lab","repos":["alice/.."]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, + {"dot repo segment", `{"name":"lab","repos":["alice/."]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, + {"dot-dot owner segment", `{"name":"lab","repos":["../lab"]}`, http.StatusBadRequest, selfServiceRefusedInvalidRepos}, + {"leading hyphen name", `{"name":"-lab","repos":["alice/lab"]}`, http.StatusBadRequest, selfServiceRefusedInvalidName}, {"http dashboard", `{"name":"lab","repos":["alice/lab"],"dashboard_url":"http://lab.example.org"}`, http.StatusBadRequest, selfServiceRefusedInvalidURL}, {"private dashboard", `{"name":"lab","repos":["alice/lab"],"dashboard_url":"https://10.0.0.5/"}`, http.StatusBadRequest, selfServiceRefusedInvalidURL}, {"bad json", `{`, http.StatusBadRequest, ""}, @@ -459,6 +469,135 @@ func TestSelfServiceAPIPerUserCap(t *testing.T) { } } +// TestSelfServiceAPIPerUserCapConcurrent pins #18166: the handler's cap count +// runs before the GitHub ownership calls, so two POSTs from one user at cap-1 +// can both pass it. The stub holds both requests inside the ownership proof +// until each has passed the handler's count; the store's own check under its +// lock must still let exactly one through. +func TestSelfServiceAPIPerUserCapConcurrent(t *testing.T) { + env := newSelfServiceTestEnv(t) + t.Setenv(selfServiceMaxPerUserEnv, "2") + if rec := env.register(t, selfServiceTestUser, `{"name":"one","repos":["alice/lab"]}`); rec.Code != http.StatusCreated { + t.Fatalf("register one: %d %s", rec.Code, rec.Body.String()) + } + + var arrived sync.WaitGroup + arrived.Add(2) + env.repoCheckGate = func() { + arrived.Done() + all := make(chan struct{}) + go func() { arrived.Wait(); close(all) }() + select { + case <-all: + case <-time.After(10 * time.Second): + // A request never reached the stub; the counts below report it. + } + } + codes := make([]int, 2) + var done sync.WaitGroup + for i, name := range []string{"two", "three"} { + done.Add(1) + go func(i int, name string) { + defer done.Done() + codes[i] = env.register(t, selfServiceTestUser, fmt.Sprintf(`{"name":%q,"repos":["alice/lab"]}`, name)).Code + }(i, name) + } + done.Wait() + + created, capped := 0, 0 + for _, c := range codes { + switch c { + case http.StatusCreated: + created++ + case http.StatusTooManyRequests: + capped++ + } + } + if created != 1 || capped != 1 { + t.Fatalf("concurrent registers at cap-1: codes %v, want exactly one 201 and one 429", codes) + } + if n := env.s.selfServiceCount(selfServiceTestUser); n != 2 { + t.Fatalf("store holds %d of alice's hives, want 2 (the cap)", n) + } + if !strings.Contains(env.logs.String(), "reason="+selfServiceRefusedCap) { + t.Fatal("no cap audit line for the store-level refusal") + } +} + +// TestSelfServiceAPIHyphenLoginsDoNotCollide pins #18166: with a single '-' +// separator, login a-b + name c and login a + name b-c both produced +// byo-a-b-c, so one user could squat the other's ID. The "--" separator keeps +// them apart. +func TestSelfServiceAPIHyphenLoginsDoNotCollide(t *testing.T) { + env := newSelfServiceTestEnv(t) + encrypted, err := encryptToken(selfServiceTestToken) + if err != nil { + t.Fatal(err) + } + for _, login := range []string{"a", "a-b"} { + if err := saveSaaSUser(&SaaSUser{GitHubUsername: login, EncryptedToken: encrypted}); err != nil { + t.Fatal(err) + } + } + ids := map[string]string{} + for _, tc := range []struct{ login, name, want string }{ + {"a", "b-c", "byo-a--b-c"}, + {"a-b", "c", "byo-a-b--c"}, + } { + rec := env.register(t, tc.login, fmt.Sprintf(`{"name":%q,"repos":["alice/lab"]}`, tc.name)) + if rec.Code != http.StatusCreated { + t.Fatalf("%s/%s: status %d body %s", tc.login, tc.name, rec.Code, rec.Body.String()) + } + var resp struct { + Hive selfServiceHiveView `json:"hive"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Hive.ID != tc.want { + t.Fatalf("%s/%s: id = %q, want %q", tc.login, tc.name, resp.Hive.ID, tc.want) + } + ids[tc.login] = resp.Hive.ID + } + for login, id := range ids { + if h, ok := env.s.selfService.get(id); !ok || !selfServiceOwns(h, login) { + t.Fatalf("%s does not own %s: %+v", login, id, h) + } + } +} + +func TestSelfServiceBuildIDRefusesAmbiguousInput(t *testing.T) { + for _, tc := range []struct { + login, name, reason string + }{ + {"a--b", "lab", selfServiceRefusedInvalidLogin}, + {"a-", "lab", selfServiceRefusedInvalidLogin}, + {"-a", "lab", selfServiceRefusedInvalidLogin}, + {"a/b", "lab", selfServiceRefusedInvalidLogin}, + {"alice", "-lab", selfServiceRefusedInvalidName}, + } { + if id, reason, err := selfServiceBuildID(tc.login, tc.name); err == nil || reason != tc.reason { + t.Errorf("selfServiceBuildID(%q, %q) = %q, %q, %v; want refusal %s", tc.login, tc.name, id, reason, err, tc.reason) + } + } + if id, _, err := selfServiceBuildID("Alice-Smith", "Lab--Two"); err != nil || id != "byo-alice-smith--lab--two" { + t.Fatalf("selfServiceBuildID(Alice-Smith, Lab--Two) = %q, %v", id, err) + } +} + +func TestIsValidRepoRefRefusesDotSegments(t *testing.T) { + for _, ref := range []string{".", "..", "owner/.", "owner/..", "./repo", "../repo"} { + if isValidRepoRef(ref) { + t.Errorf("isValidRepoRef(%q) = true, want false", ref) + } + } + for _, ref := range []string{"owner/.github", "owner/a.b", "owner/...", ".github"} { + if !isValidRepoRef(ref) { + t.Errorf("isValidRepoRef(%q) = false, want true", ref) + } + } +} + func TestSelfServiceAPIRotateInvalidatesOldKey(t *testing.T) { env := newSelfServiceTestEnv(t) id, oldKey := env.registerHappy(t) @@ -535,7 +674,7 @@ func TestSelfServiceAPIAnotherUsersHiveIsNotFound(t *testing.T) { if rec := env.remove(t, selfServiceTestOther, id); rec.Code != http.StatusNotFound { t.Fatalf("delete by other user: %d, want 404", rec.Code) } - if rec := env.rotate(t, selfServiceTestUser, "byo-alice-nope"); rec.Code != http.StatusNotFound { + if rec := env.rotate(t, selfServiceTestUser, "byo-alice--nope"); rec.Code != http.StatusNotFound { t.Fatalf("rotate of missing hive: %d, want 404", rec.Code) } if _, ok := env.s.verifyHeartbeatBearerGeneration(key, id); !ok { diff --git a/src/pkg/hub/selfservice_registry.go b/src/pkg/hub/selfservice_registry.go index ee749bd5a5..8cd0bd4b35 100644 --- a/src/pkg/hub/selfservice_registry.go +++ b/src/pkg/hub/selfservice_registry.go @@ -69,6 +69,9 @@ var ( errSelfServiceNotFound = errors.New("self-service hive not found") errSelfServiceNilStore = errors.New("self-service store not configured") errSelfServiceEmptyOwner = errors.New("self-service hive owner required") + // errSelfServiceCapReached is returned by registerCapped when the owner + // already holds the maximum number of hives (#18166). + errSelfServiceCapReached = errors.New("self-service per-owner cap reached") ) // SelfServiceHive is one owner-registered hive. Only hashes of its keys are @@ -207,11 +210,22 @@ func hashSelfServiceKey(raw string) string { return hex.EncodeToString(sum[:]) } -// register adds a new self-service hive and returns its raw key ONCE. The ID -// must already be in the byo- namespace; callers (the registration API) build -// it as byo-<login>-<name> and run the registry-collision and validNamePattern -// checks before calling. +// register adds a new self-service hive with no per-owner cap and returns its +// raw key ONCE. See registerCapped. func (st *selfServiceStore) register(h SelfServiceHive) (string, error) { + return st.registerCapped(h, 0) +} + +// registerCapped adds a new self-service hive and returns its raw key ONCE. +// The ID must already be in the byo- namespace; callers (the registration API) +// build it with selfServiceBuildID and run the registry-collision and +// validNamePattern checks before calling. +// +// When maxPerOwner > 0, the owner's existing hives are counted under the same +// write lock that inserts the new one, so concurrent registrations by one +// owner can never exceed the cap (#18166): the handler's earlier count is only +// a fast refusal before the GitHub ownership calls, and it races. +func (st *selfServiceStore) registerCapped(h SelfServiceHive, maxPerOwner int) (string, error) { if st == nil { return "", errSelfServiceNilStore } @@ -231,6 +245,17 @@ func (st *selfServiceStore) register(h SelfServiceHive) (string, error) { if _, exists := st.hives[h.ID]; exists { return "", errSelfServiceExists } + if maxPerOwner > 0 { + owned := 0 + for _, existing := range st.hives { + if selfServiceOwns(*existing, h.Owner) { + owned++ + } + } + if owned >= maxPerOwner { + return "", errSelfServiceCapReached + } + } h.KeyHash = hash h.PrevKeyHash = "" h.PrevKeyExpiresAt = time.Time{} diff --git a/src/pkg/hub/selfservice_registry_test.go b/src/pkg/hub/selfservice_registry_test.go index 695b0216d2..e9cc148da0 100644 --- a/src/pkg/hub/selfservice_registry_test.go +++ b/src/pkg/hub/selfservice_registry_test.go @@ -2,12 +2,15 @@ package hub import ( "bytes" + "errors" + "fmt" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -339,3 +342,41 @@ func TestSelfServiceStoreStampPersistIsDebounced(t *testing.T) { t.Fatalf("unknown ID stamp returned an error: %v", err) } } + +// TestSelfServiceStoreRegisterCappedConcurrent pins #18166 at the store: many +// concurrent registrations by one owner never exceed maxPerOwner, and another +// owner's hives do not count against it. +func TestSelfServiceStoreRegisterCappedConcurrent(t *testing.T) { + st := newSelfServiceStore(filepath.Join(t.TempDir(), "hub-selfservice.json")) + if _, err := st.registerCapped(SelfServiceHive{ID: "byo-bob--lab", Owner: "bob"}, 3); err != nil { + t.Fatal(err) + } + const attempts, limit = 10, 3 + errs := make([]error, attempts) + var wg sync.WaitGroup + for i := 0; i < attempts; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, errs[i] = st.registerCapped(SelfServiceHive{ID: fmt.Sprintf("byo-alice--h%d", i), Owner: "alice"}, limit) + }(i) + } + wg.Wait() + ok, capped := 0, 0 + for _, err := range errs { + switch { + case err == nil: + ok++ + case errors.Is(err, errSelfServiceCapReached): + capped++ + default: + t.Fatalf("unexpected error: %v", err) + } + } + if ok != limit || capped != attempts-limit { + t.Fatalf("registered %d, capped %d; want %d and %d", ok, capped, limit, attempts-limit) + } + if n := len(st.list()); n != limit+1 { + t.Fatalf("store holds %d hives, want %d", n, limit+1) + } +} diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 79338bc5c1..94ec9a0f1b 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -940,7 +940,10 @@ func validateSingleRepoHost(spokeHost, primaryRepo string, repos []string) error // isValidRepoRef validates a repo entry, which may be a bare name ("repo") // or a cross-org "owner/repo" reference. Both segments must be safe names -// (no path traversal); at most one slash is allowed. Cross-org repos are a +// (no path traversal); at most one slash is allowed. "." and ".." pass the +// safe-name charset but are dot segments that url.JoinPath and friends +// resolve (owner/.. would become a request for /repos), and GitHub allows +// neither as a repo or owner name, so both are refused (#18166). Cross-org repos are a // supported hive config (e.g. a primary repo in one org plus a secondary in // another), so the heartbeat must not 400-reject them — doing so silently // took a whole hive offline (no heartbeats → no upgrades, shows offline). @@ -953,7 +956,7 @@ func isValidRepoRef(s string) bool { return false } for _, p := range parts { - if !isValidName(p) { + if !isValidName(p) || p == "." || p == ".." { return false } } From 6a8ad5c28f185e1ff6009984777dff1660b2271c Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 21:11:44 -0500 Subject: [PATCH 200/223] =?UTF-8?q?=E2=9C=A8=20reference=20repos:=20resolv?= =?UTF-8?q?e=20derived=20entries=20from=20the=20App=20installation,=20read?= =?UTF-8?q?-only=20fallback=20with=20one=20marked=20comment=20(#18139=20st?= =?UTF-8?q?ep=202)=20(#18171)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ reference repos: resolve derived entries from the App installation, read-only fallback with one marked comment (#18158) A project.reference_repos entry derived from an accepted request issue (#18139) is not the operator's mistake, so it must never refuse boot. github.ResolveDerivedReferenceRepo keeps the write tier when the App has an installation on the target org and otherwise clears Write in memory with CredentialState read-only and a reason; a transient discovery failure is served read-only as "unresolved" without telling anyone. write_token_file is never consulted for a derived entry. Hand-written entries keep the exit-22 verification exactly. github.Client.CommentDerivedReadOnly posts one marked <!-- hive:reference-repos-read-only repos=... --> comment per (request issue, repo set) through EditIssueMarkerComment, so a reload never reposts. The resolved entries replace the in-memory list, and /api/status carries derived_from, credential_state and credential_reason for #18161. Part of #18139. Signed-off-by: Andy Anderson <andy@clubanderson.com> * 📖 api-reference: refresh server.go citations shifted by the reference repo fields (#18158) Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...8158-derived-reference-repo-credentials.md | 1 + src/cmd/hive/reference_repos.go | 119 +++++++++-- src/cmd/hive/reference_repos_test.go | 186 +++++++++++++++++- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +-- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/github-write-surface.md | 1 + src/pkg/config/reference_repos.go | 46 +++++ src/pkg/config/reference_repos_test.go | 54 +++++ .../reference_repos_status_18021_test.go | 53 +++++ src/pkg/dashboard/server.go | 8 + src/pkg/dashboard/status_builder.go | 5 +- src/pkg/github/reference_repo_request.go | 117 +++++++++++ src/pkg/github/reference_repo_request_test.go | 94 +++++++++ src/pkg/github/reference_repo_token.go | 69 ++++++- src/pkg/github/reference_repo_token_test.go | 96 +++++++++ src/pkg/github/write_surface_internal.go | 5 + 18 files changed, 848 insertions(+), 38 deletions(-) create mode 100644 changelog.d/added-18158-derived-reference-repo-credentials.md create mode 100644 src/pkg/github/reference_repo_request.go create mode 100644 src/pkg/github/reference_repo_request_test.go diff --git a/changelog.d/added-18158-derived-reference-repo-credentials.md b/changelog.d/added-18158-derived-reference-repo-credentials.md new file mode 100644 index 0000000000..7853a169cf --- /dev/null +++ b/changelog.d/added-18158-derived-reference-repo-credentials.md @@ -0,0 +1 @@ +- A `project.reference_repos` entry derived from an accepted request issue resolves its credential from the GitHub App installation on the requesting org instead of refusing to boot: with an installation it keeps `write: [issues, prs]`, without one it is served read-only in memory and the request issue gets exactly one marked `hive:reference-repos-read-only` comment saying the hive can read but not open issues or PRs there (re-posted only when the read-only set changes, never on a plain reload). `write_token_file` is never consulted for a derived entry, a hand-written write-mode entry without a credential still exits 22, and `/api/status` `referenceRepos` entries now carry `derived_from`, `credential_state` (`write-tier`, `read-only`, `unresolved`) and `credential_reason` (#18139, #18158). diff --git a/src/cmd/hive/reference_repos.go b/src/cmd/hive/reference_repos.go index f87c367245..a812de7adb 100644 --- a/src/cmd/hive/reference_repos.go +++ b/src/cmd/hive/reference_repos.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "os" + "sort" "github.com/hivecommons/hive/crux/proxy" "github.com/hivecommons/hive/pkg/config" @@ -19,6 +20,11 @@ import ( // write_token_file", not "bug". const referenceRepoCredentialExitCode = 22 +// derivedReadOnlyPoster posts the one-shot read-only notice on a request +// issue (hivecommons/hive#18158). Production is +// github.Client.CommentDerivedReadOnly; tests stub it. +type derivedReadOnlyPoster func(ctx context.Context, issueRepo string, number int, readOnly []config.ReferenceRepo) (bool, error) + // referenceRepoDeps are the seams buildReferenceRepoPolicies needs: App // installation discovery on a foreign org and token-identity resolution. // Either may be nil when the hive has no GitHub App auth, in which case a @@ -30,35 +36,113 @@ type referenceRepoDeps struct { auth func() *github.AppAuth } +// referenceRepoResolution is what buildReferenceRepoPolicies produces: the +// proxy policies and the entries as resolved, with each derived entry's +// CredentialState filled in (and its Write cleared when it was downgraded), +// so the kick and the status payload describe what the proxy enforces. +type referenceRepoResolution struct { + policies []proxy.ReferenceRepoPolicy + repos []config.ReferenceRepo +} + // buildReferenceRepoPolicies turns the config list into proxy policies. A -// read-only entry needs nothing; a write-mode entry is verified now so the -// failure is a boot-time config error naming the repo. -func buildReferenceRepoPolicies(ctx context.Context, refs []config.ReferenceRepo, deps referenceRepoDeps, logger *slog.Logger) ([]proxy.ReferenceRepoPolicy, error) { - out := make([]proxy.ReferenceRepoPolicy, 0, len(refs)) +// read-only entry needs nothing; a hand-written write-mode entry is verified +// now so the failure is a boot-time config error naming the repo. A derived +// entry (hivecommons/hive#18158) is resolved instead: write tier when the App +// is installed on its org, read-only otherwise, never an error (a transient +// discovery failure keeps the entry read-only for this load and is logged). +func buildReferenceRepoPolicies(ctx context.Context, refs []config.ReferenceRepo, deps referenceRepoDeps, logger *slog.Logger) (referenceRepoResolution, error) { + res := referenceRepoResolution{ + policies: make([]proxy.ReferenceRepoPolicy, 0, len(refs)), + repos: make([]config.ReferenceRepo, 0, len(refs)), + } for _, rr := range refs { - pol := proxy.ReferenceRepoPolicy{Repo: rr.Repo, Issues: rr.WritesIssues(), PRs: rr.WritesPRs()} - if !rr.ReadOnly() { - login, err := github.VerifyReferenceRepoWriteCredential(ctx, rr, deps.botLogin, deps.discover, deps.whoAmI) + login := "" + if rr.Derived() { + resolved, derivedLogin, err := github.ResolveDerivedReferenceRepo(ctx, rr, deps.botLogin, deps.discover) if err != nil { - return nil, err + // A blip, not a verdict: serve read-only this load but do not + // tell the request issue the App is missing when it may not be. + logger.Warn("derived reference repo: credential discovery failed, serving read-only this load", "repo", rr.Repo, "from", rr.DerivedFrom, "error", err.Error()) + resolved.Write = nil + resolved.CredentialState = config.ReferenceCredentialUnresolved + resolved.CredentialReason = err.Error() } + rr, login = resolved, derivedLogin + } else if !rr.ReadOnly() { + l, err := github.VerifyReferenceRepoWriteCredential(ctx, rr, deps.botLogin, deps.discover, deps.whoAmI) + if err != nil { + return referenceRepoResolution{}, err + } + login = l + } + pol := proxy.ReferenceRepoPolicy{Repo: rr.Repo, Issues: rr.WritesIssues(), PRs: rr.WritesPRs()} + if !rr.ReadOnly() { pol.Login = login pol.Token = github.ReferenceRepoTokenSource(rr, deps.auth, logger) } - out = append(out, pol) + res.policies = append(res.policies, pol) + res.repos = append(res.repos, rr) + } + return res, nil +} + +// derivedReadOnlyByRequest groups the downgraded derived entries by the +// request issue that granted them, in a stable order of issue refs. +func derivedReadOnlyByRequest(repos []config.ReferenceRepo) (refs []string, byRef map[string][]config.ReferenceRepo) { + byRef = map[string][]config.ReferenceRepo{} + for _, rr := range repos { + if !rr.DerivedReadOnly() { + continue + } + if _, ok := byRef[rr.DerivedFrom]; !ok { + refs = append(refs, rr.DerivedFrom) + } + byRef[rr.DerivedFrom] = append(byRef[rr.DerivedFrom], rr) + } + sort.Strings(refs) + return refs, byRef +} + +// notifyDerivedReadOnly posts the read-only notice for every request issue +// whose derived entries were downgraded. The poster is idempotent per +// (issue, repo set), so calling this on every reload posts at most once. +func notifyDerivedReadOnly(ctx context.Context, repos []config.ReferenceRepo, post derivedReadOnlyPoster, logger *slog.Logger) { + if post == nil { + return + } + refs, byRef := derivedReadOnlyByRequest(repos) + for _, ref := range refs { + readOnly := byRef[ref] + issueRepo, number, ok := readOnly[0].DerivedFromIssue() + if !ok { + logger.Warn("derived reference repo: malformed request issue ref, notice skipped", "from", ref) + continue + } + posted, err := post(ctx, issueRepo, number, readOnly) + if err != nil { + logger.Warn("derived reference repo: read-only notice not posted", "from", ref, "error", err.Error()) + continue + } + if posted { + logger.Info("derived reference repos served read-only, request issue notified", "from", ref, "count", len(readOnly)) + } } - return out, nil } // wireReferenceRepos installs project.reference_repos on the proxy, refusing -// to boot when a write-mode entry has no usable credential. +// to boot when a hand-written write-mode entry has no usable credential. A +// derived entry never refuses boot: it is downgraded to read-only and the +// request issue is told once (hivecommons/hive#18158). func (b *boot) wireReferenceRepos() { refs := b.cfg.Project.ReferenceRepos if len(refs) == 0 || b.githubProxy == nil { return } deps := referenceRepoDeps{auth: func() *github.AppAuth { return nil }} + var post derivedReadOnlyPoster if b.ghClient != nil { + post = b.ghClient.CommentDerivedReadOnly if auth := b.ghClient.AppAuth(); auth != nil { apiURL := auth.APIURL() deps.botLogin = auth.BotLogin() @@ -74,7 +158,7 @@ func (b *boot) wireReferenceRepos() { } } } - policies, err := buildReferenceRepoPolicies(b.ctx, refs, deps, b.logger) + res, err := buildReferenceRepoPolicies(b.ctx, refs, deps, b.logger) if err != nil { b.logger.Error("refusing to start: project.reference_repos write-mode entry has no usable credential (#18015)", "error", err.Error()) if errors.Is(err, github.ErrReferenceRepoNoCredential) { @@ -82,6 +166,13 @@ func (b *boot) wireReferenceRepos() { } os.Exit(referenceRepoCredentialExitCode) } - b.githubProxy.SetReferenceRepos(policies) - b.logger.Info("reference repos wired", "count", len(policies), "repos", b.cfg.ReferenceRepoSlugs()) + // The resolved entries replace the list in memory only: a derived entry + // is never written back to any config file, and its credential state is + // what the kick and /api/status read. + b.cfg.Project.ReferenceRepos = res.repos + b.githubProxy.SetReferenceRepos(res.policies) + b.logger.Info("reference repos wired", "count", len(res.policies), "repos", b.cfg.ReferenceRepoSlugs()) + // Off the boot path: the notice is a GitHub write and must not delay + // the proxy coming up. + go notifyDerivedReadOnly(b.ctx, res.repos, post, b.logger) } diff --git a/src/cmd/hive/reference_repos_test.go b/src/cmd/hive/reference_repos_test.go index b10d70646a..bdadd91718 100644 --- a/src/cmd/hive/reference_repos_test.go +++ b/src/cmd/hive/reference_repos_test.go @@ -3,15 +3,21 @@ package main import ( "context" "errors" + "strings" + "sync" "testing" + "time" + "github.com/hivecommons/hive/internal/testutil" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" ) -func TestBuildReferenceRepoPolicies(t *testing.T) { - ctx := context.Background() - deps := referenceRepoDeps{ +// referenceRepoTestTimeout bounds the waits on the async notice poster. +const referenceRepoTestTimeout = 5 * time.Second + +func testReferenceRepoDeps() referenceRepoDeps { + return referenceRepoDeps{ botLogin: "hive[bot]", discover: func(_ context.Context, org string) (int64, error) { if org == "hivecommons" { @@ -21,16 +27,22 @@ func TestBuildReferenceRepoPolicies(t *testing.T) { }, auth: func() *github.AppAuth { return nil }, } +} + +func TestBuildReferenceRepoPolicies(t *testing.T) { + ctx := context.Background() + deps := testReferenceRepoDeps() refs := []config.ReferenceRepo{ {Repo: "hivecommons/docs"}, {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, } - pols, err := buildReferenceRepoPolicies(ctx, refs, deps, testLogger()) + res, err := buildReferenceRepoPolicies(ctx, refs, deps, testLogger()) if err != nil { t.Fatal(err) } - if len(pols) != 2 { - t.Fatalf("got %d policies", len(pols)) + pols := res.policies + if len(pols) != 2 || len(res.repos) != 2 { + t.Fatalf("got %d policies, %d repos", len(pols), len(res.repos)) } if pols[0].Token != nil || pols[0].Issues || pols[0].PRs || pols[0].Login != "" { t.Errorf("read-only policy carries write state: %+v", pols[0]) @@ -38,9 +50,171 @@ func TestBuildReferenceRepoPolicies(t *testing.T) { if pols[1].Token == nil || !pols[1].Issues || !pols[1].PRs || pols[1].Login != "hive[bot]" { t.Errorf("write policy incomplete: %+v", pols[1]) } + for i, rr := range res.repos { + if rr.CredentialState != "" || rr.Repo != refs[i].Repo { + t.Errorf("hand-written entry %d must come back unchanged: %+v", i, rr) + } + } + // A hand-written write-mode entry without a credential still refuses + // boot with the exit-22 error text. _, err = buildReferenceRepoPolicies(ctx, []config.ReferenceRepo{{Repo: "kubernetes/website", Write: []string{"issues"}}}, deps, testLogger()) if !errors.Is(err, github.ErrReferenceRepoNoCredential) { t.Fatalf("uninstalled org must refuse with ErrReferenceRepoNoCredential, got %v", err) } + if !strings.Contains(err.Error(), "kubernetes/website") || !strings.Contains(err.Error(), "needs the GitHub App installed on kubernetes") { + t.Fatalf("exit-22 error must name the repo and the cure: %v", err) + } + if referenceRepoCredentialExitCode != 22 { + t.Fatalf("exit code = %d, want 22", referenceRepoCredentialExitCode) + } +} + +// notifyRecorder stubs the request-issue notice poster. +type notifyRecorder struct { + mu sync.Mutex + calls []notifyCall + // posted is what the stub reports back; the real poster says false when + // the marker already exists. + posted bool + err error +} + +type notifyCall struct { + repo string + number int + repos []string +} + +func (n *notifyRecorder) post(_ context.Context, issueRepo string, number int, readOnly []config.ReferenceRepo) (bool, error) { + n.mu.Lock() + defer n.mu.Unlock() + call := notifyCall{repo: issueRepo, number: number} + for _, rr := range readOnly { + call.repos = append(call.repos, rr.Repo) + } + n.calls = append(n.calls, call) + return n.posted, n.err +} + +func (n *notifyRecorder) count() int { + n.mu.Lock() + defer n.mu.Unlock() + return len(n.calls) +} + +// hivecommons/hive#18158: a derived entry never refuses boot. With the App +// installed on its org it keeps the write tier; without, it is served +// read-only and the request issue is told once. +func TestBuildReferenceRepoPoliciesDerived(t *testing.T) { + ctx := context.Background() + deps := testReferenceRepoDeps() + refs := []config.ReferenceRepo{ + {Repo: "hivecommons/docs", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395"}, + {Repo: "acme/docs", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395"}, + {Repo: "acme/site", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395"}, + {Repo: "other/repo", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#400"}, + {Repo: "hivecommons/hive"}, + } + res, err := buildReferenceRepoPolicies(ctx, refs, deps, testLogger()) + if err != nil { + t.Fatalf("a derived entry on an uninstalled org must not refuse boot: %v", err) + } + if len(res.policies) != len(refs) || len(res.repos) != len(refs) { + t.Fatalf("got %d policies, %d repos, want %d", len(res.policies), len(res.repos), len(refs)) + } + writeTier := res.repos[0] + if !writeTier.WritesIssues() || !writeTier.WritesPRs() || writeTier.CredentialState != config.ReferenceCredentialWriteTier { + t.Errorf("installed org must keep the write tier: %+v", writeTier) + } + if p := res.policies[0]; !p.Issues || !p.PRs || p.Login != "hive[bot]" || p.Token == nil { + t.Errorf("write-tier policy incomplete: %+v", p) + } + for i := 1; i <= 3; i++ { + rr, p := res.repos[i], res.policies[i] + if !rr.DerivedReadOnly() || !rr.ReadOnly() || rr.CredentialReason == "" { + t.Errorf("uninstalled org must downgrade to read-only: %+v", rr) + } + if p.Issues || p.PRs || p.Token != nil || p.Login != "" { + t.Errorf("read-only policy carries write state: %+v", p) + } + if rr.DerivedFrom != refs[i].DerivedFrom { + t.Errorf("provenance lost: %+v", rr) + } + } + if res.repos[4].CredentialState != "" { + t.Errorf("hand-written read-only entry must carry no credential state: %+v", res.repos[4]) + } + + // First load: one notice per request issue, each with its read-only set. + rec := ¬ifyRecorder{posted: true} + go notifyDerivedReadOnly(ctx, res.repos, rec.post, testLogger()) + testutil.Eventually(t, referenceRepoTestTimeout, func() bool { return rec.count() == 2 }, "want 2 notices, got %d", rec.count()) + rec.mu.Lock() + calls := append([]notifyCall(nil), rec.calls...) + rec.mu.Unlock() + byIssue := map[int][]string{} + for _, c := range calls { + if c.repo != "cncf/techdocs" { + t.Errorf("notice posted to %q, want the request issue's repo", c.repo) + } + byIssue[c.number] = c.repos + } + if got := strings.Join(byIssue[395], ","); got != "acme/docs,acme/site" { + t.Errorf("#395 read-only set = %q", got) + } + if got := strings.Join(byIssue[400], ","); got != "other/repo" { + t.Errorf("#400 read-only set = %q", got) + } + + // Second reload with the same outcome: the poster finds its marker and + // reports nothing posted; the wiring neither errors nor loops. + rec2 := ¬ifyRecorder{posted: false} + res2, err := buildReferenceRepoPolicies(ctx, refs, deps, testLogger()) + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + notifyDerivedReadOnly(ctx, res2.repos, rec2.post, testLogger()) + close(done) + }() + testutil.Eventually(t, referenceRepoTestTimeout, func() bool { + select { + case <-done: + return true + default: + return false + } + }, "second reload notify did not finish") + if rec2.count() != 2 { + t.Fatalf("second reload asked the poster %d times, want 2 (the poster decides idempotency)", rec2.count()) + } + + // A poster failure is logged, never fatal; a nil poster is a no-op. + rec3 := ¬ifyRecorder{err: errors.New("boom")} + notifyDerivedReadOnly(ctx, res.repos, rec3.post, testLogger()) + notifyDerivedReadOnly(ctx, res.repos, nil, testLogger()) +} + +// A transient discovery failure must not downgrade a derived entry with a +// "not installed" verdict: it is served read-only this load as unresolved +// and the request issue is not told. +func TestBuildReferenceRepoPoliciesDerivedTransientError(t *testing.T) { + deps := testReferenceRepoDeps() + deps.discover = func(context.Context, string) (int64, error) { return 0, errors.New("dial tcp: i/o timeout") } + refs := []config.ReferenceRepo{{Repo: "acme/docs", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395"}} + res, err := buildReferenceRepoPolicies(context.Background(), refs, deps, testLogger()) + if err != nil { + t.Fatalf("transient error must not refuse boot: %v", err) + } + rr := res.repos[0] + if !rr.ReadOnly() || rr.CredentialState != config.ReferenceCredentialUnresolved || rr.DerivedReadOnly() { + t.Fatalf("got %+v", rr) + } + rec := ¬ifyRecorder{posted: true} + notifyDerivedReadOnly(context.Background(), res.repos, rec.post, testLogger()) + if rec.count() != 0 { + t.Fatalf("an unresolved entry must not notify the request issue, got %d calls", rec.count()) + } } diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index d2c9ff2407..0386da1826 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1726` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1727` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1728` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1734` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1735` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1736` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index fee159b157..e9ff3b72f0 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1733` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1741` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 49e5b0e099..9f95dec44d 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1706` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1707` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1708` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1714` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1719` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1720` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1721` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1722` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1723` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1724` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1725` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1714` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1715` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1716` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1722` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1727` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1728` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1729` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1730` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1731` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1732` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1733` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index b06253c0ef..658d675f9c 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1796` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1804` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index 61ebcbd711..cc6144b8d4 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -162,6 +162,7 @@ writes no entry. | Fleet report recovery | `fleet_report.go` | yes (`fleet_report_recovered`, with `closed`) | yes / issue number | | Legacy standalone `Fleet report:` close (hourly sweep in the hive watching the report repo, #13470) | `fleet_report_legacy_sweep.go` | yes (`fleet_report_recovered`, `reason=legacy_standalone`) | yes / issue number | | Reporter-trust wait notice/comment and owned label cleanup | `client.go` / `reporter_trust_wait.go` (`fetchIssues` reporter-trust reject/admit paths) | yes (`reporter_trust_wait_noticed`, `reporter_trust_wait_cleared`; label adds also record `hive_label_applied` with `reason=reporter_trust_wait`) | yes / issue number | +| Derived reference repos read-only notice (one `hive:reference-repos-read-only` comment per request issue and repo set, #18158) | `reference_repo_request.go` (`CommentDerivedReadOnly`) | yes (`reference_repos_read_only_noticed`, with `repos`) | yes / issue number | The open-time signed rewrite in `pr_request_signed.go` is part of `open_pr` and is covered by that operation's `agent_pr_created` entry. Creating a diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go index 1d36d47265..ebee601355 100644 --- a/src/pkg/config/reference_repos.go +++ b/src/pkg/config/reference_repos.go @@ -3,6 +3,7 @@ package config import ( "fmt" "sort" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -45,12 +46,57 @@ type ReferenceRepo struct { // an overlay or the runtime config, where it would outlive the grant // (see hivecommons/hive#18137). The dashboard reads it as JSON. DerivedFrom string `yaml:"-" json:"derived_from,omitempty"` + // CredentialState is the resolved credential tier of a derived entry + // (hivecommons/hive#18158): ReferenceCredentialWriteTier when the App is + // installed on the target org, ReferenceCredentialReadOnly when it is not + // and the entry was downgraded in memory. Empty for a hand-written entry, + // whose credential is verified at boot instead. Never serialized to YAML: + // it is re-resolved on every load. + CredentialState string `yaml:"-" json:"credential_state,omitempty"` + // CredentialReason says why CredentialState is read-only, in words the + // request issue comment and the Settings editor can show. + CredentialReason string `yaml:"-" json:"credential_reason,omitempty"` } +// Credential states of a derived reference repo. +const ( + // ReferenceCredentialWriteTier: the App has an installation on the target + // org, the entry keeps its issues+prs opt-in. + ReferenceCredentialWriteTier = "write-tier" + // ReferenceCredentialReadOnly: no installation, the entry was downgraded + // to read-only in memory and the request issue told why. + ReferenceCredentialReadOnly = "read-only" + // ReferenceCredentialUnresolved: installation discovery failed for a + // transient reason (network, 5xx); the entry is served read-only for + // this load without telling the request issue, and is retried on the + // next reload. + ReferenceCredentialUnresolved = "unresolved" +) + // Derived reports whether the entry came from an accepted request issue // rather than from the operator's config. func (r ReferenceRepo) Derived() bool { return r.DerivedFrom != "" } +// DerivedReadOnly reports whether a derived entry was downgraded to +// read-only because no credential covers it. +func (r ReferenceRepo) DerivedReadOnly() bool { + return r.Derived() && r.CredentialState == ReferenceCredentialReadOnly +} + +// DerivedFromIssue splits DerivedFrom ("owner/repo#N") into the request +// issue's repo and number; ok is false for a hand-written or malformed entry. +func (r ReferenceRepo) DerivedFromIssue() (repo string, number int, ok bool) { + repo, num, found := strings.Cut(strings.TrimSpace(r.DerivedFrom), "#") + if !found || !strings.Contains(repo, "/") { + return "", 0, false + } + n, err := strconv.Atoi(num) + if err != nil || n <= 0 { + return "", 0, false + } + return repo, n, true +} + const ( ReferenceWriteIssues = "issues" ReferenceWritePRs = "prs" diff --git a/src/pkg/config/reference_repos_test.go b/src/pkg/config/reference_repos_test.go index fce0389522..77430ba058 100644 --- a/src/pkg/config/reference_repos_test.go +++ b/src/pkg/config/reference_repos_test.go @@ -184,3 +184,57 @@ func TestReferenceRepoDerivedFromNeverSerializesToYAML(t *testing.T) { t.Fatalf("JSON must carry derived_from for the dashboard:\n%s", js) } } + +// hivecommons/hive#18158: the resolved credential state is in-memory only, +// like the provenance it belongs to; JSON carries it for the dashboard. +func TestReferenceRepoCredentialStateNeverSerializesToYAML(t *testing.T) { + rr := ReferenceRepo{Repo: "acme/docs", DerivedFrom: "cncf/techdocs#395", CredentialState: ReferenceCredentialReadOnly, CredentialReason: "the hive's GitHub App is not installed on acme"} + if !rr.DerivedReadOnly() { + t.Fatal("read-only derived entry must read as DerivedReadOnly") + } + if (ReferenceRepo{Repo: "acme/docs", CredentialState: ReferenceCredentialReadOnly}).DerivedReadOnly() { + t.Fatal("a hand-written entry is never DerivedReadOnly") + } + if (ReferenceRepo{Repo: "acme/docs", DerivedFrom: "x/y#1", CredentialState: ReferenceCredentialWriteTier}).DerivedReadOnly() { + t.Fatal("a write-tier entry is not DerivedReadOnly") + } + for _, entry := range []ReferenceRepo{rr, {Repo: "acme/docs", Write: []string{ReferenceWriteIssues}, DerivedFrom: "cncf/techdocs#395", CredentialState: ReferenceCredentialWriteTier}} { + out, err := yaml.Marshal(entry) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "credential") || strings.Contains(string(out), "installed") { + t.Fatalf("credential state leaked into YAML:\n%s", out) + } + } + js, err := json.Marshal(rr) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(js), `"credential_state":"read-only"`) || !strings.Contains(string(js), `"credential_reason":`) { + t.Fatalf("JSON must carry the credential state:\n%s", js) + } +} + +func TestReferenceRepoDerivedFromIssue(t *testing.T) { + cases := []struct { + from string + repo string + number int + ok bool + }{ + {"cncf/techdocs#395", "cncf/techdocs", 395, true}, + {" cncf/techdocs#7 ", "cncf/techdocs", 7, true}, + {"", "", 0, false}, + {"cncf/techdocs", "", 0, false}, + {"techdocs#395", "", 0, false}, + {"cncf/techdocs#0", "", 0, false}, + {"cncf/techdocs#abc", "", 0, false}, + } + for _, tc := range cases { + repo, number, ok := (ReferenceRepo{DerivedFrom: tc.from}).DerivedFromIssue() + if repo != tc.repo || number != tc.number || ok != tc.ok { + t.Errorf("DerivedFromIssue(%q) = %q, %d, %v; want %q, %d, %v", tc.from, repo, number, ok, tc.repo, tc.number, tc.ok) + } + } +} diff --git a/src/pkg/dashboard/reference_repos_status_18021_test.go b/src/pkg/dashboard/reference_repos_status_18021_test.go index ca6dde9572..1291a7b9b5 100644 --- a/src/pkg/dashboard/reference_repos_status_18021_test.go +++ b/src/pkg/dashboard/reference_repos_status_18021_test.go @@ -77,5 +77,58 @@ func TestStatusReferenceReposCarryTierAndNoManagedFields(t *testing.T) { t.Errorf("%s carries managed-repo field %q: %v", full, k, rr) } } + for _, k := range []string{"derived_from", "credential_state", "credential_reason"} { + if _, present := rr[k]; present { + t.Errorf("hand-written %s carries derived field %q: %v", full, k, rr) + } + } + } +} + +// hivecommons/hive#18158: a derived entry's resolved credential state rides +// along in /api/status as credential_state (plus reason and provenance), so +// the Settings editor can show write-tier against read-only. +func TestStatusReferenceReposCarryCredentialState(t *testing.T) { + srv := newFullServer(t) + cfg := srv.deps.Config + cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs", Write: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier}, + {Repo: "acme/docs", DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialReadOnly, CredentialReason: "the hive's GitHub App is not installed on acme"}, + } + srv.UpdateStatus(&StatusPayload{ReferenceRepos: buildReferenceRepos(cfg)}) + + rec := doGet(srv, "/api/status") + if rec.Code != http.StatusOK { + t.Fatalf("/api/status = %d %s", rec.Code, rec.Body.String()) + } + var body struct { + ReferenceRepos []map[string]any `json:"referenceRepos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode /api/status: %v", err) + } + if len(body.ReferenceRepos) != 2 { + t.Fatalf("referenceRepos = %v, want both entries", body.ReferenceRepos) + } + want := map[string]map[string]any{ + "hivecommons/docs": {"reference_tier": "issues+prs", "credential_state": "write-tier", "derived_from": "cncf/techdocs#395"}, + "acme/docs": {"reference_tier": "read-only", "credential_state": "read-only", "derived_from": "cncf/techdocs#395", "credential_reason": "the hive's GitHub App is not installed on acme"}, + } + for _, rr := range body.ReferenceRepos { + full, _ := rr["full"].(string) + fields, ok := want[full] + if !ok { + t.Fatalf("unexpected reference entry %v", rr) + } + for k, v := range fields { + if got := rr[k]; got != v { + t.Errorf("%s %s = %v, want %v", full, k, got, v) + } + } + if full == "hivecommons/docs" { + if _, present := rr["credential_reason"]; present { + t.Errorf("write-tier entry must carry no credential_reason: %v", rr) + } + } } } diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 748d82fbbf..ddd90a1e0f 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -1242,6 +1242,14 @@ type FrontendReferenceRepo struct { // ReferenceTier repeats Tier under the name API consumers key on to tell // a reference entry from a managed repo row (hivecommons/hive#18021). ReferenceTier string `json:"reference_tier"` + // DerivedFrom is the "owner/repo#N" request issue a derived entry came + // from (hivecommons/hive#18139); empty for a hand-written entry. + DerivedFrom string `json:"derived_from,omitempty"` + // CredentialState is a derived entry's resolved credential: + // "write-tier", "read-only" or "unresolved" (hivecommons/hive#18158); + // CredentialReason says why when it is not write-tier. + CredentialState string `json:"credential_state,omitempty"` + CredentialReason string `json:"credential_reason,omitempty"` } type FrontendRepo struct { diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 15040fa8a4..2a5186c7fd 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -2386,7 +2386,10 @@ func buildReferenceRepos(cfg *config.Config) []FrontendReferenceRepo { out := make([]FrontendReferenceRepo, 0, len(cfg.Project.ReferenceRepos)) for _, rr := range cfg.Project.ReferenceRepos { tier := rr.WriteLabel() - out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: tier, ReferenceTier: tier}) + out = append(out, FrontendReferenceRepo{ + Full: rr.Repo, Tier: tier, ReferenceTier: tier, + DerivedFrom: rr.DerivedFrom, CredentialState: rr.CredentialState, CredentialReason: rr.CredentialReason, + }) } return out } diff --git a/src/pkg/github/reference_repo_request.go b/src/pkg/github/reference_repo_request.go new file mode 100644 index 0000000000..dfd0be6f4d --- /dev/null +++ b/src/pkg/github/reference_repo_request.go @@ -0,0 +1,117 @@ +package github + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// Request-issue notice for derived reference repos that could not get a +// write credential (hivecommons/hive#18158). +// +// A derived entry (config.ReferenceRepo.DerivedFrom) was granted by a +// maintainer accepting a request issue in one of the hive's authorized repos. +// When the App is not installed on the requesting project's org the hive can +// still read the repos but cannot open issues or PRs there. That is the +// requesting project's call to make, not the operator's, so the hive says so +// once on the request issue instead of refusing to boot. + +// DerivedReadOnlyMarkerPrefix opens the marker the notice carries. The +// complete marker names the read-only repo set, so one notice is posted per +// (issue, repo set): a reload with the same outcome finds it and posts +// nothing, a later grant that adds a repo posts a new one. +const DerivedReadOnlyMarkerPrefix = "<!-- hive:reference-repos-read-only " + +// derivedReadOnlyMarkerSuffix closes the marker. +const derivedReadOnlyMarkerSuffix = " -->" + +// derivedReadOnlyRepoSeparator joins the repo slugs inside the marker. +const derivedReadOnlyRepoSeparator = "," + +// DerivedReadOnlyMarker is the full marker for a read-only repo set, with +// the slugs case-folded and sorted so the set compares as a set. +func DerivedReadOnlyMarker(repos []string) string { + slugs := make([]string, 0, len(repos)) + seen := make(map[string]bool, len(repos)) + for _, r := range repos { + slug := strings.ToLower(strings.TrimSpace(r)) + if slug == "" || seen[slug] { + continue + } + seen[slug] = true + slugs = append(slugs, slug) + } + sort.Strings(slugs) + return DerivedReadOnlyMarkerPrefix + "repos=" + strings.Join(slugs, derivedReadOnlyRepoSeparator) + + " state=" + config.ReferenceCredentialReadOnly + derivedReadOnlyMarkerSuffix +} + +// DerivedReadOnlyComment renders the notice body for the read-only entries +// derived from one request issue. It lists each repo with its reason and +// says what the hive can and cannot do; it names no accounts, so nobody is +// notified beyond the issue's subscribers. +func DerivedReadOnlyComment(readOnly []config.ReferenceRepo) string { + if len(readOnly) == 0 { + return "" + } + repos := make([]string, 0, len(readOnly)) + for _, rr := range readOnly { + repos = append(repos, rr.Repo) + } + var b strings.Builder + b.WriteString(DerivedReadOnlyMarker(repos)) + b.WriteString("\n") + if len(readOnly) == 1 { + b.WriteString("This request grants the hive access to one repository it can read but not write:\n") + } else { + b.WriteString("This request grants the hive access to repositories it can read but not write:\n") + } + for _, rr := range readOnly { + reason := strings.TrimSpace(rr.CredentialReason) + if reason == "" { + reason = DerivedReadOnlyReason(rr) + } + fmt.Fprintf(&b, "- `%s`: %s\n", rr.Repo, reason) + } + b.WriteString("\nThe hive will study these repositories (clone, read files, read issues and PRs) and keep its findings in the assessment deliverable here. ") + b.WriteString("It will not open issues or pull requests on them. ") + b.WriteString("To let it, an owner of that organization installs the same GitHub App there; the hive picks the installation up on its next config reload without a restart.") + return AppendTrailer(b.String(), hiveWriteMeta()) +} + +// CommentDerivedReadOnly posts the read-only notice for one request issue +// (issueRepo#number, an authorized repo of this hive) once per repo set: +// when a comment of the hive's carrying the same marker exists nothing is +// posted and posted is false. readOnly are the derived entries from that +// issue whose CredentialState is read-only; an empty list posts nothing. +func (c *Client) CommentDerivedReadOnly(ctx context.Context, issueRepo string, number int, readOnly []config.ReferenceRepo) (posted bool, err error) { + if c == nil { + return false, ErrNoGitHubClient + } + if len(readOnly) == 0 { + return false, nil + } + repos := make([]string, 0, len(readOnly)) + for _, rr := range readOnly { + repos = append(repos, rr.Repo) + } + marker := DerivedReadOnlyMarker(repos) + res, err := c.EditIssueMarkerComment(ctx, issueRepo, number, marker, func(previous string) string { + if previous != "" { + // Same repo set already noticed: leave the existing comment alone. + return previous + } + return DerivedReadOnlyComment(readOnly) + }) + if err != nil { + return false, fmt.Errorf("derived reference repos read-only notice on %s#%d: %w", issueRepo, number, err) + } + if res.Created { + c.recordWriteAudit(AuditActionReferenceRepoReadOnlyNoticed, hiveWriteMeta(), + WriteTarget{Repo: c.qualifiedRepo(issueRepo), Number: number}, "repos", strings.Join(repos, derivedReadOnlyRepoSeparator)) + } + return res.Created, nil +} diff --git a/src/pkg/github/reference_repo_request_test.go b/src/pkg/github/reference_repo_request_test.go new file mode 100644 index 0000000000..c7f8a23744 --- /dev/null +++ b/src/pkg/github/reference_repo_request_test.go @@ -0,0 +1,94 @@ +package github + +import ( + "context" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +func derivedReadOnlyFixture() []config.ReferenceRepo { + return []config.ReferenceRepo{ + {Repo: "acme/docs", DerivedFrom: "o/r#7", CredentialState: config.ReferenceCredentialReadOnly, CredentialReason: "the hive's GitHub App is not installed on acme"}, + {Repo: "acme/site", DerivedFrom: "o/r#7", CredentialState: config.ReferenceCredentialReadOnly, CredentialReason: "the hive's GitHub App is not installed on acme"}, + } +} + +func TestDerivedReadOnlyMarkerIsASetKey(t *testing.T) { + a := DerivedReadOnlyMarker([]string{"Acme/Site", "acme/docs", " acme/docs "}) + b := DerivedReadOnlyMarker([]string{"acme/docs", "acme/site"}) + if a != b { + t.Fatalf("marker must not depend on order, case or duplicates:\n%s\n%s", a, b) + } + if !strings.HasPrefix(a, DerivedReadOnlyMarkerPrefix) || !strings.Contains(a, "repos=acme/docs,acme/site") || !strings.Contains(a, "state=read-only") || !strings.HasSuffix(a, "-->") { + t.Fatalf("marker = %q", a) + } + if DerivedReadOnlyMarker([]string{"acme/docs"}) == a { + t.Fatal("a different repo set must yield a different marker") + } +} + +func TestDerivedReadOnlyCommentBody(t *testing.T) { + body := DerivedReadOnlyComment(derivedReadOnlyFixture()) + for _, want := range []string{DerivedReadOnlyMarkerPrefix, "`acme/docs`", "`acme/site`", "not installed on acme", "will not open issues or pull requests", AttributionTrailerPrefix} { + if !strings.Contains(body, want) { + t.Errorf("comment lacks %q:\n%s", want, body) + } + } + if strings.Contains(body, "@") { + t.Fatalf("comment must not mention anyone:\n%s", body) + } + if DerivedReadOnlyComment(nil) != "" { + t.Fatal("no read-only entries must render nothing") + } +} + +// hivecommons/hive#18158: exactly one notice per (request issue, repo set), +// however many reloads happen; a wider set posts a second one. +func TestCommentDerivedReadOnlyPostsOnce(t *testing.T) { + s := &statusCommentServer{} + srv := s.start(t) + c := testClient(t, srv.URL) + c.SetAppBotLogin("hive-app[bot]") + ctx := context.Background() + readOnly := derivedReadOnlyFixture() + + posted, err := c.CommentDerivedReadOnly(ctx, "o/r", 7, readOnly) + if err != nil || !posted { + t.Fatalf("first notice posted=%v err=%v", posted, err) + } + for i := 0; i < 3; i++ { + posted, err = c.CommentDerivedReadOnly(ctx, "o/r", 7, readOnly) + if err != nil || posted { + t.Fatalf("reload %d reposted: posted=%v err=%v", i, posted, err) + } + } + if s.creates != 1 || s.edits != 0 || len(s.comments) != 1 { + t.Fatalf("creates=%d edits=%d comments=%d, want 1/0/1", s.creates, s.edits, len(s.comments)) + } + body, _ := s.comments[0]["body"].(string) + if !strings.Contains(body, DerivedReadOnlyMarker([]string{"acme/docs", "acme/site"})) { + t.Fatalf("comment body lacks the marker:\n%s", body) + } + + // Same set in another order is the same notice. + posted, err = c.CommentDerivedReadOnly(ctx, "o/r", 7, []config.ReferenceRepo{readOnly[1], readOnly[0]}) + if err != nil || posted || s.creates != 1 { + t.Fatalf("reordered set reposted: posted=%v err=%v creates=%d", posted, err, s.creates) + } + // A later grant that adds a repo is a new repo set and a new notice. + wider := append(readOnly, config.ReferenceRepo{Repo: "acme/legacy", DerivedFrom: "o/r#7", CredentialState: config.ReferenceCredentialReadOnly}) + posted, err = c.CommentDerivedReadOnly(ctx, "o/r", 7, wider) + if err != nil || !posted || s.creates != 2 { + t.Fatalf("wider set: posted=%v err=%v creates=%d", posted, err, s.creates) + } + // Nothing read-only, nothing posted. + posted, err = c.CommentDerivedReadOnly(ctx, "o/r", 7, nil) + if err != nil || posted || s.creates != 2 { + t.Fatalf("empty set: posted=%v err=%v creates=%d", posted, err, s.creates) + } + if _, err := (*Client)(nil).CommentDerivedReadOnly(ctx, "o/r", 7, readOnly); err == nil { + t.Fatal("nil client must refuse") + } +} diff --git a/src/pkg/github/reference_repo_token.go b/src/pkg/github/reference_repo_token.go index 7000ddfc47..67974c6e58 100644 --- a/src/pkg/github/reference_repo_token.go +++ b/src/pkg/github/reference_repo_token.go @@ -66,7 +66,9 @@ func ReferenceRepoTokenSource(rr config.ReferenceRepo, auth func() *AppAuth, log if rr.ReadOnly() { return nil } - if rr.WriteTokenFile != "" { + // A derived entry never reads write_token_file (hivecommons/hive#18158): + // its only credential is the App installation on the target org. + if rr.WriteTokenFile != "" && !rr.Derived() { return func() (string, bool) { tok, err := readTokenFile(rr.WriteTokenFile) if err != nil { @@ -170,6 +172,71 @@ func VerifyReferenceRepoWriteCredential(ctx context.Context, rr config.Reference return botLogin, nil } +// Derived read-only reasons (hivecommons/hive#18158). They are shown to the +// requesting project on its request issue, so they name the cure, not the +// hive's internals. +const ( + // derivedReadOnlyNoInstallationFmt is used when the App is not installed + // on the target org; the argument is the owner. + derivedReadOnlyNoInstallationFmt = "the hive's GitHub App is not installed on %s" + // DerivedReadOnlyNoAppAuth is used when the hive runs without App auth at + // all, so no installation can ever be found. + DerivedReadOnlyNoAppAuth = "this hive has no GitHub App auth" + // DerivedReadOnlyBotLoginUnknown is used when an installation exists but + // the hive cannot name its own bot login, so it could not tell its own + // items from others' and must not write. + DerivedReadOnlyBotLoginUnknown = "the hive's App bot login is unknown" +) + +// DerivedReadOnlyReason is the reason recorded on a derived entry whose +// target org has no installation of the App. +func DerivedReadOnlyReason(rr config.ReferenceRepo) string { + return fmt.Sprintf(derivedReadOnlyNoInstallationFmt, rr.Owner()) +} + +// ResolveDerivedReferenceRepo resolves the credential for an entry derived +// from a request issue (hivecommons/hive#18158). Unlike a hand-written entry, +// a derived one is not the operator's mistake, so it never yields +// ErrReferenceRepoNoCredential: when the App has an installation on the +// target org the entry is returned unchanged with CredentialState write-tier +// and the bot login the credential acts as; when there is none the entry +// comes back with Write cleared, CredentialState read-only and the reason. +// write_token_file is never consulted for a derived entry. A transient +// discovery failure (network, 5xx) is returned as an error so the caller +// keeps its last good state instead of downgrading on a blip. A hand-written +// entry is returned unchanged with no login; use +// VerifyReferenceRepoWriteCredential for it. +func ResolveDerivedReferenceRepo(ctx context.Context, rr config.ReferenceRepo, botLogin string, discover func(ctx context.Context, org string) (int64, error)) (config.ReferenceRepo, string, error) { + if !rr.Derived() { + return rr, "", nil + } + rr.WriteTokenFile = "" + readOnly := func(reason string) (config.ReferenceRepo, string, error) { + rr.Write = nil + rr.CredentialState = config.ReferenceCredentialReadOnly + rr.CredentialReason = reason + return rr, "", nil + } + if rr.ReadOnly() { + return readOnly(rr.CredentialReason) + } + if discover == nil { + return readOnly(DerivedReadOnlyNoAppAuth) + } + if _, err := discover(ctx, rr.Owner()); err != nil { + if errors.Is(err, ErrNoInstallationForOrg) { + return readOnly(DerivedReadOnlyReason(rr)) + } + return rr, "", fmt.Errorf("resolving derived reference repo %s (from %s): %w", rr.Repo, rr.DerivedFrom, err) + } + if botLogin == "" { + return readOnly(DerivedReadOnlyBotLoginUnknown) + } + rr.CredentialState = config.ReferenceCredentialWriteTier + rr.CredentialReason = "" + return rr, botLogin, nil +} + // TokenUserLogin resolves the login a user token acts as (GET /user). func TokenUserLogin(ctx context.Context, token, apiURL string) (string, error) { user, _, err := newTokenClient(token, apiURL).Users.Get(ctx, "") diff --git a/src/pkg/github/reference_repo_token_test.go b/src/pkg/github/reference_repo_token_test.go index 0024f336aa..570dd46580 100644 --- a/src/pkg/github/reference_repo_token_test.go +++ b/src/pkg/github/reference_repo_token_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "github.com/hivecommons/hive/pkg/config" @@ -72,6 +73,101 @@ func TestVerifyReferenceRepoWriteCredential(t *testing.T) { } } +// hivecommons/hive#18158: a derived entry resolves against the App +// installation on its org and never fails like a hand-written one. +func TestResolveDerivedReferenceRepo(t *testing.T) { + ctx := context.Background() + netErr := errors.New("dial tcp: i/o timeout") + installed := func(_ context.Context, org string) (int64, error) { + switch org { + case "hivecommons": + return 42, nil + case "flaky": + return 0, netErr + } + return 0, ErrNoInstallationForOrg + } + derived := func(repo string) config.ReferenceRepo { + return config.ReferenceRepo{Repo: repo, Write: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}, DerivedFrom: "cncf/techdocs#395"} + } + + t.Run("installation found keeps write tier", func(t *testing.T) { + got, login, err := ResolveDerivedReferenceRepo(ctx, derived("hivecommons/docs"), "hive[bot]", installed) + if err != nil || login != "hive[bot]" { + t.Fatalf("login=%q err=%v", login, err) + } + if !got.WritesIssues() || !got.WritesPRs() || got.CredentialState != config.ReferenceCredentialWriteTier || got.CredentialReason != "" { + t.Fatalf("resolved = %+v", got) + } + if !got.Derived() || got.DerivedReadOnly() { + t.Fatalf("provenance lost or misread: %+v", got) + } + }) + t.Run("no installation downgrades to read-only", func(t *testing.T) { + got, login, err := ResolveDerivedReferenceRepo(ctx, derived("acme/docs"), "hive[bot]", installed) + if err != nil || login != "" { + t.Fatalf("login=%q err=%v (a derived entry must never error on a missing installation)", login, err) + } + if !got.ReadOnly() || !got.DerivedReadOnly() || got.CredentialState != config.ReferenceCredentialReadOnly { + t.Fatalf("resolved = %+v", got) + } + if got.CredentialReason != DerivedReadOnlyReason(derived("acme/docs")) || !strings.Contains(got.CredentialReason, "acme") { + t.Fatalf("reason = %q", got.CredentialReason) + } + if got.DerivedFrom != "cncf/techdocs#395" { + t.Fatalf("provenance lost: %+v", got) + } + }) + t.Run("write_token_file is never consulted", func(t *testing.T) { + rr := derived("acme/docs") + rr.WriteTokenFile = filepath.Join(t.TempDir(), "nope") + got, _, err := ResolveDerivedReferenceRepo(ctx, rr, "hive[bot]", installed) + if err != nil || got.WriteTokenFile != "" || !got.DerivedReadOnly() { + t.Fatalf("got %+v err=%v", got, err) + } + if ReferenceRepoTokenSource(rr, func() *AppAuth { return nil }, testLogger()) == nil { + t.Fatal("derived write-mode entry must have a token source") + } + if _, ok := ReferenceRepoTokenSource(rr, func() *AppAuth { return nil }, testLogger())(); ok { + t.Fatal("derived entry must not read write_token_file") + } + }) + t.Run("no app auth is read-only", func(t *testing.T) { + got, _, err := ResolveDerivedReferenceRepo(ctx, derived("hivecommons/docs"), "", nil) + if err != nil || !got.DerivedReadOnly() || got.CredentialReason != DerivedReadOnlyNoAppAuth { + t.Fatalf("got %+v err=%v", got, err) + } + }) + t.Run("bot login unknown is read-only", func(t *testing.T) { + got, _, err := ResolveDerivedReferenceRepo(ctx, derived("hivecommons/docs"), "", installed) + if err != nil || !got.DerivedReadOnly() || got.CredentialReason != DerivedReadOnlyBotLoginUnknown { + t.Fatalf("got %+v err=%v", got, err) + } + }) + t.Run("transient discovery error propagates", func(t *testing.T) { + got, _, err := ResolveDerivedReferenceRepo(ctx, derived("flaky/docs"), "hive[bot]", installed) + if !errors.Is(err, netErr) { + t.Fatalf("want the network error, got %v", err) + } + if errors.Is(err, ErrReferenceRepoNoCredential) { + t.Fatal("a derived entry must never yield ErrReferenceRepoNoCredential") + } + if !strings.Contains(err.Error(), "flaky/docs") || !strings.Contains(err.Error(), "cncf/techdocs#395") { + t.Fatalf("error must name the repo and the request: %v", err) + } + if !got.WritesIssues() { + t.Fatal("the entry comes back unchanged for the caller to keep its last good state") + } + }) + t.Run("hand-written entry passes through", func(t *testing.T) { + rr := config.ReferenceRepo{Repo: "acme/docs", Write: []string{"issues"}} + got, login, err := ResolveDerivedReferenceRepo(ctx, rr, "hive[bot]", installed) + if err != nil || login != "" || got.CredentialState != "" || !got.WritesIssues() { + t.Fatalf("got %+v login=%q err=%v", got, login, err) + } + }) +} + func TestReferenceRepoTokenSource_File(t *testing.T) { tokFile := filepath.Join(t.TempDir(), "tok") if err := os.WriteFile(tokFile, []byte("first\n"), 0o600); err != nil { diff --git a/src/pkg/github/write_surface_internal.go b/src/pkg/github/write_surface_internal.go index ed558ee9f2..2daa02a996 100644 --- a/src/pkg/github/write_surface_internal.go +++ b/src/pkg/github/write_surface_internal.go @@ -72,6 +72,10 @@ const ( // AuditActionReporterTrustWaitCleared is recorded when the core poller // removes a needs-triage label that its wait marker proves Hive added. AuditActionReporterTrustWaitCleared = "reporter_trust_wait_cleared" + // AuditActionReferenceRepoReadOnlyNoticed is recorded when boot or a + // config reload posts the one-shot notice that derived reference repos + // are served read-only (reference_repo_request.go, hivecommons/hive#18158). + AuditActionReferenceRepoReadOnlyNoticed = "reference_repos_read_only_noticed" ) // Outcome values for the internal-writer audit entries' "outcome" pair. @@ -106,5 +110,6 @@ func InternalWriteAuditActions() []string { AuditActionHiveIDLabelRetired, AuditActionReporterTrustWaitNoticed, AuditActionReporterTrustWaitCleared, + AuditActionReferenceRepoReadOnlyNoticed, } } From 80e9ac48434efd26fceaae5ec09bbd7d23b3a633 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 21:27:29 -0500 Subject: [PATCH 201/223] =?UTF-8?q?=E2=9C=A8=20marketplace:=20cncf-techdoc?= =?UTF-8?q?s-assessment=20curated-2026.11.2=20declares=20intake.referenceR?= =?UTF-8?q?epoFields=20(#18169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second curated TechDocs model file, identical to curated-2026.11.1 except for spec.intake, which names the rendered assessment-request.yml headings ("Website or documentation repository", "Other repositories or sites in scope", out of scope: "Out of scope") whose repositories become derived write-tier reference repos once a maintainer accepts the request (hivecommons/hive#18139 step 4). Curated files are never edited in place, so 2026.11.1 stays as first published and a hive still bound to it derives nothing. Tests: the curated gate loads both versions and pins that they differ only in version, annotations and intake; the hub seed publishes both with 2026.11.2 last in the listing; refderive derives cncf/techdocs#395 through the embedded 2026.11.2 model and refuses through 2026.11.1. Fixes #18160 Part of #18139 Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-18160-techdocs-model-intake.md | 1 + src/docs/workflows/techdocs-assessment.md | 38 +- .../cncf-techdocs-assessment-2026.11.2.yaml | 652 ++++++++++++++++++ .../hub/marketplace/curated/curated_test.go | 96 ++- src/pkg/hub/marketplace_curated_test.go | 57 +- src/pkg/refderive/derive_curated_test.go | 67 ++ 6 files changed, 879 insertions(+), 32 deletions(-) create mode 100644 changelog.d/added-18160-techdocs-model-intake.md create mode 100644 src/pkg/hub/marketplace/curated/cncf-techdocs-assessment-2026.11.2.yaml create mode 100644 src/pkg/refderive/derive_curated_test.go diff --git a/changelog.d/added-18160-techdocs-model-intake.md b/changelog.d/added-18160-techdocs-model-intake.md new file mode 100644 index 0000000000..b25e3492e7 --- /dev/null +++ b/changelog.d/added-18160-techdocs-model-intake.md @@ -0,0 +1 @@ +- The hub marketplace seeds a second curated TechDocs model, `cncf-techdocs-assessment@curated-2026.11.2`, identical to `curated-2026.11.1` except for a `spec.intake` block naming the assessment-request form headings ("Website or documentation repository", "Other repositories or sites in scope", "Out of scope") that yield derived write-tier reference repos once a maintainer accepts the request; `curated-2026.11.1` stays in the catalogue unchanged, so a hive still bound to it derives nothing until it rebinds (#18139, #18160). diff --git a/src/docs/workflows/techdocs-assessment.md b/src/docs/workflows/techdocs-assessment.md index a849b775eb..236c1620bd 100644 --- a/src/docs/workflows/techdocs-assessment.md +++ b/src/docs/workflows/techdocs-assessment.md @@ -1,8 +1,8 @@ # CNCF TechDocs AI-assisted assessment — curated workflow model -`cncf-techdocs-assessment@curated-2026.11.1` is the first curated model in the hub -marketplace (https://hive.hivecommons.dev → Settings → Workflows → Marketplace on any -hive). It encodes the CNCF TechDocs team's AI-assisted documentation assessment process +`cncf-techdocs-assessment@curated-2026.11.2` is the current version of the first curated +model in the hub marketplace (https://hive.hivecommons.dev → Settings → Workflows → +Marketplace on any hive); `curated-2026.11.1` stays in the catalogue unchanged. It encodes the CNCF TechDocs team's AI-assisted documentation assessment process so a hive bound to `cncf/techdocs` runs intake, drafting, verification, stakeholder review and approval as a visible lifecycle, with every agent action followed by a human gate. @@ -18,10 +18,12 @@ annotations and in each element's `citations`): `pkg/workflowmodel/ingest/testdata/techdocs/`. The model's scope is `github.com cncf/techdocs` — the upstream repository where the -process runs; the fork above is where the spec was written. Model file: -`pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml` +process runs; the fork above is where the spec was written. Model files: +`pkg/hub/marketplace/curated/cncf-techdocs-assessment-2026.11.2.yaml` (current) and +`pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml` (`curated-2026.11.1`) (`apiVersion: workflow.hivecommons.dev/v1alpha1`, `derivation.mode: rules`, -`meta.provenance: curated`). The same repository is also in the ingest corpus +`meta.provenance: curated`). Curated files are never edited in place: a change is a new +file with a new `meta.version`, and the catalogue keeps every version. The same repository is also in the ingest corpus (`pkg/workflowmodel/ingest/corpus/extra.json`), so an `ingested-unreviewed` version built from the pipeline files alone sits next to this one in the catalogue. @@ -132,6 +134,30 @@ domains → the agent's egress allowlist; contacts → the frozen stakeholder se at `/accept`); tooling/context → drafter input; the HC-6 disclosure acknowledgement is required. +#### Reference repos from the request + +`curated-2026.11.2` adds a `spec.intake` block +([hivecommons/hive#18139](https://github.com/hivecommons/hive/issues/18139)): + +```yaml +intake: + referenceRepoFields: + - "Website or documentation repository" + - "Other repositories or sites in scope" + outOfScopeFields: + - "Out of scope" +``` + +The entries are the rendered `### <label>` headings of `assessment-request.yml` as they +appear in a request issue's body, not the form's field ids. A hive bound to this version +derives one write-tier (`issues+prs`) `project.reference_repos` entry per repository named +under a `referenceRepoFields` heading and not under an `outOfScopeFields` heading, once a +maintainer with write access to `cncf/techdocs` applies `triage/accepted` to the request. +The derived entries carry the request as provenance (`cncf/techdocs#395` style), are +rebuilt on every config load and never written to the overlay or runtime config, and +disappear when the issue closes or the label is removed. `curated-2026.11.1` is unchanged +and declares no intake fields, so a hive that stays bound to it derives nothing. + ## Not representable today These parts of the spec are documented in the model's annotations but not enforced by diff --git a/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment-2026.11.2.yaml b/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment-2026.11.2.yaml new file mode 100644 index 0000000000..5ffae90c63 --- /dev/null +++ b/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment-2026.11.2.yaml @@ -0,0 +1,652 @@ +# Curated marketplace model: the CNCF TechDocs AI-assisted assessment +# lifecycle (hivecommons/hive#17620), version curated-2026.11.2. Identical to +# curated-2026.11.1 (cncf-techdocs-assessment.yaml) except for the +# spec.intake block (hivecommons/hive#18139): it names the rendered headings +# of the assessment-request form whose values are repositories the hive may +# derive as write-tier reference repos once a maintainer accepts the request. +# A hive still bound to 2026.11.1 derives nothing. A new version is a new +# file; this one is never edited in place. +apiVersion: workflow.hivecommons.dev/v1alpha1 +kind: WorkflowModel +metadata: + name: cncf-techdocs-assessment + labels: + workflow.hivecommons.dev/tier: curated + annotations: + workflow.hivecommons.dev/source: nate-double-u/cncf-techdocs@882be6fef28794433bc3a6a34ac1401391c1d3e4 specs/ai-assisted-techdocs-assessment-spec.md + workflow.hivecommons.dev/source-url: https://github.com/nate-double-u/cncf-techdocs/blob/ai-assisted-assessment-spec/specs/ai-assisted-techdocs-assessment-spec.md + workflow.hivecommons.dev/intake-template: nate-double-u/cncf-techdocs@ad915303f9774b9aa6b7646969897fada7975a00 .github/ISSUE_TEMPLATE/assessment-request.yml + workflow.hivecommons.dev/intake-template-url: https://github.com/nate-double-u/cncf-techdocs/blob/ad915303f9774b9aa6b7646969897fada7975a00/.github/ISSUE_TEMPLATE/assessment-request.yml + # Intake form field id -> the fact or record it feeds (spec §7, §12 Accept). + workflow.hivecommons.dev/intake-fields: "project-name=assessment.project; maturity=assessment.maturity; moving-levels=assessment.criteria-level; website,docs-url,repo,other-scope,out-of-scope=assessment.scope; domains=agent.egress-allowlist; contacts=stakeholder frozen set (validated at /accept); tooling,context=drafter input; disclosure=HC-6 acknowledgement (required)" + workflow.hivecommons.dev/curated-by: Andrew Anderson <andy@clubanderson.com> + workflow.hivecommons.dev/curated-from: ingest.Lifecycle(cncf/techdocs .github snapshot) + spec.Ingest(deterministic) skeleton, then hand-curated + workflow.hivecommons.dev/spec-sections: "4. Roles; 5. Lifecycle; 6. Phases and deliverables; 7. Inputs: the assessment request; 12. Lifecycle bindings; 13. Components and repository layout; 14. Agent definitions; 15. The provenance block" + # What changed since curated-2026.11.1: the spec.intake block declares which + # intake form headings name reference repositories (hivecommons/hive#18139). + workflow.hivecommons.dev/changes: "curated-2026.11.2: adds spec.intake.referenceRepoFields and spec.intake.outOfScopeFields so an accepted request issue yields derived write-tier reference repos (hivecommons/hive#18139); lifecycle unchanged from curated-2026.11.1" +spec: + meta: + version: curated-2026.11.2 + author: CNCF TechDocs (Nate W.); curated for Hive by Andrew Anderson + description: >- + AI-assisted, expert-supervised CNCF TechDocs assessment: an intake issue is + triaged and accepted by a technical writer, an agent drafts each phase's + deliverable as a draft PR, a verifier fact-checks it, the reviewer marks it + ready, the project's stakeholders confirm factual accuracy and an + independent approver merges. Phases A/B/C repeat the same six steps and are + carried by the phase/* labels. + license: CC-BY-4.0 + source: spec + provenance: curated + scope: + forge: github.com + repository: cncf/techdocs + workTypes: [documentation-assessment] + # Intake hook (hivecommons/hive#18139): the rendered "### <label>" headings of + # assessment-request.yml whose values name repositories. Once a maintainer with + # write access applies triage/accepted, the hive derives one write-tier + # (issues+prs) reference repo per repository named under referenceRepoFields + # and not under outOfScopeFields. Labels, not field ids: the body of a request + # issue shows the label. + intake: + referenceRepoFields: + - "Website or documentation repository" + - "Other repositories or sites in scope" + outOfScopeFields: + - "Out of scope" + sources: + - system: github-actions + path: .github/workflows/format-check.yml + sha256: bf8cbb5896ad92f1e88f1051998450b35d0c080b27dba24f03c5d58d54e4a67e + - system: github-actions + path: .github/workflows/link-check.yml + sha256: 1bcef2093eda4ef4ca90d5766985e7710fee8926e0e1d85e5e2055593e679e94 + - system: github-actions + path: .github/workflows/spell-check.yml + sha256: d144101194175c7382e093d176019815fbe460c3633ef892350bc2a2635d7f87 + initial: requested + columns: + - id: requested + label: Requested + hint: Intake or phase tracking issue filed; a technical writer triages it (spec §5 step 1, §12 Request) + states: [requested] + - id: accepted + label: Accepted + hint: A technical writer accepted (/accept) and owns the phase as reviewer; delegate the drafter (spec §5 step 2) + states: [accepted, escalated] + - id: drafting + label: Drafting + hint: AI-drafted deliverable as a draft PR; verifier pass runs here (spec §5 step 3, §12 Draft) + states: [draft-open, drafting] + - id: review + label: Reviewer check + hint: Verified; the reviewer refines in conversation and checks findings against source (spec §5 step 4, HC-7) + states: [in-review, checks-failing] + - id: stakeholders + label: Stakeholder review + hint: Ready for review; project stakeholders confirm factual accuracy with /confirm (spec §5 step 5, HC-2) + states: [stakeholder-review] + - id: approval + label: Approval + hint: Confirmed; an approver who was neither drafter nor reviewer signs off and merges (spec §5 step 6) + states: [awaiting-merge] + - id: done + label: Done + hint: Merged, discarded, declined or closed (spec §12 Merge, Failure and abort) + states: [merged, discarded, declined, closed] + terminal: true + states: + # ---- issues: the intake (umbrella) issue and per-phase tracking issues ---- + - id: requested + label: Requested + owner: human + origin: human + lifecycle: + column: requested + number: 10 + kinds: [issue] + waiting: [awaiting triage by a technical writer] + replies: ["/accept", "/decline"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Request. Phase A is triggered by the intake issue; the next phase becomes eligible once the previous one is merged" + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "A request issue opens labeled needs-triage ... An issue may sit unassigned indefinitely; eligibility is not work." + - path: .github/ISSUE_TEMPLATE/assessment-request.yml + line: 4 + text: "labels: Docs analysis, needs-triage" + - id: accepted + label: Accepted + owner: human + origin: human + lifecycle: + column: accepted + number: 20 + kinds: [issue] + waiting: [reviewer to delegate the phase drafter, or restart/hand-write after a discard] + replies: ["/decline"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Accept. A technical writer triages and explicitly accepts the request ... eligibility alone does not start work (P-1)" + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/accept ... assigns the issue to them, recording who accepted and who acts as the phase's reviewer, and swaps needs-triage for triage/accepted" + - path: .github/settings.yml + line: 132 + text: triage/accepted + - id: escalated + label: Escalated + owner: human + origin: human + lifecycle: + column: accepted + number: 45 + kinds: [issue] + waiting: [administrator/platform owner to abort or release the hold] + replies: ["/abort"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Repeated failure is escalated to the administrator/platform owner, who may abort the run." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "An administrator/platform owner abort closes the tracking issue and the intake with it as not planned, the rationale recorded on both" + - id: draft-open + label: Draft under way + owner: registered-agent + origin: human + lifecycle: + column: drafting + number: 40 + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "Draft (step 3): a draft pull request. The agent works on its own branch in cncf/techdocs and opens a draft PR linked to the tracking issue" + - id: declined + label: Declined + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "a writer declines an unaccepted request with /decline [reason] ... labels it triage/declined ... and closes the issue as not planned" + - path: .github/settings.yml + line: 135 + text: triage/declined + - id: closed + label: Closed + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The final phase's merge closes the intake issue as completed ... the close reason and labels say how it ended, completed at the final merge, not planned on a decline or an abort." + # ---- pull requests: the phase deliverable ---- + - id: drafting + label: Drafting + owner: registered-agent + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: drafting + number: 70 + kinds: [pr] + waiting: [drafter to finish, then the reviewer's /verify (or a recorded skip)] + replies: ["/verify", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Draft. The drafter produces the deliverable as a draft PR." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer triggers the verifier's pass by commenting /verify on the draft PR; its report lands as a PR comment ... the workflow applies the verified label" + - path: "spec:14. Agent definitions#L716-L792" + text: "Assessment drafter (Phase A) ... Plan drafter (Phase B) ... Backlog drafter (Phase C) ... Verifier (every phase)" + - id: in-review + label: Reviewer check + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: review + number: 50 + kinds: [pr] + waiting: [reviewer to verify findings against source and comment /ready] + replies: ["/ready", "/skip-implementation", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Review. The verifier's fact-check pass runs first ... The reviewer then refines the draft in conversation with the drafter, verifies findings against source (HC-7), and marks it ready." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The label is the gate: /ready refuses without it" + - path: .github/settings.yml + line: 147 + text: verified + - id: checks-failing + label: Checks failing + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: review + number: 71 + kinds: [pr] + waiting: [format, link or spelling check to go green] + replies: ["/discard"] + citations: + - path: .github/workflows/format-check.yml + text: Format checks + - path: .github/workflows/link-check.yml + text: Link checks + - path: .github/workflows/spell-check.yml + text: Spelling checks + - id: stakeholder-review + label: Stakeholder review + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: stakeholders + number: 60 + kinds: [pr] + waiting: [a stakeholder from the set frozen at acceptance to comment /confirm] + replies: ["/confirm", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Stakeholder review. Stakeholders confirm factual accuracy (required to advance) and record any disagreement with the conclusions (not required; HC-2)." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer comments /ready: the workflow marks the PR ready for review and mentions the stakeholders from the set frozen at acceptance" + - id: awaiting-merge + label: Awaiting approver + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: approval + number: 90 + kinds: [pr] + waiting: [an approver who was neither drafter nor reviewer of this phase to sign off and merge] + replies: ["/approve", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Merge. The approver signs off and merges the PR into cncf/techdocs, which makes the next phase eligible." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The approver merges, first checking the confirmation behind the confirmed label" + - path: .github/settings.yml + line: 150 + text: confirmed + - id: merged + label: Merged + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [pr] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "A merge-triggered GitHub Actions workflow then opens the next phase's tracking issue, labeled for its phase and needs-triage" + - id: discarded + label: Discarded + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [pr] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer discards a draft with /discard <reason>, which closes the PR with the reason recorded ... the tracking issue stays open for a restart or a hand-written phase." + roles: + - id: requester + label: Requester + kind: author-kind + params: {kind: human} + human: true + - id: reviewer + label: Reviewer (technical writer) + kind: permission + params: {permission: write} + human: true + - id: approver + label: Approver (independent technical writer) + kind: permission + params: {permission: write} + human: true + - id: stakeholder + label: Project stakeholder + kind: frozen-set + params: {frozenAt: accepted, source: "intake contacts validated at /accept"} + human: true + - id: administrator + label: Administrator / platform owner + kind: permission + params: {permission: admin} + human: true + - id: drafter + label: Phase drafter (agent) + kind: bot + params: {login: hive} + - id: verifier + label: Verifier (agent) + kind: bot + params: {login: hive} + gates: + - id: accept + releasedBy: reviewer + reply: "/accept" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "the workflow refuses /accept from a commenter without write access" + - id: decline + releasedBy: reviewer + reply: "/decline" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/decline [reason], under the same authorization as /accept" + - id: verified + conditions: + - {fact: labels, op: has, value: verified} + releasedBy: reviewer + reply: "/verify" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The label means the verification this PR needs is satisfied, not that the agent ran: a reviewer who deliberately skips the pass ... applies the label by hand with the reason in a comment" + - id: ready + conditions: + - {fact: labels, op: has, value: verified} + - {fact: pr.checks, op: is, value: green} + releasedBy: reviewer + reply: "/ready" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/ready refuses without it [the verified label]" + - id: confirm + conditions: + - {fact: labels, op: has, value: confirmed} + releasedBy: stakeholder + reply: "/confirm" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/confirm, which the workflow checks against the stakeholder set frozen at acceptance ... never the requester-editable intake body" + - id: approve + conditions: + - {fact: labels, op: has, value: confirmed} + - {fact: pr.checks, op: is, value: green} + releasedBy: approver + reply: "/approve" + citations: + - path: "spec:4. Roles#L166-L218" + text: "Approver. Gives each phase its independent quality sign-off and merges the PR. An experienced technical writer who was neither the drafter nor a reviewer of that phase" + - id: discard + releasedBy: reviewer + reply: "/discard" + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "the reviewer may discard it and restart or write by hand ... retries are chosen, never automatic" + - id: abort + releasedBy: administrator + reply: "/abort" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "An administrator/platform owner abort closes the tracking issue and the intake with it as not planned" + events: + - id: accepted + label: Request accepted + fact: {fact: labels, op: has, value: triage/accepted} + actor: reviewer + - id: declined + label: Request declined + fact: {fact: labels, op: has, value: triage/declined} + actor: reviewer + - id: draft-opened + label: Draft PR opened + fact: {fact: issue.covered, op: "true"} + - id: draft-withdrawn + label: Draft discarded; issue uncovered + fact: {fact: issue.covered, op: "false"} + - id: verified + label: Verification satisfied + fact: {fact: labels, op: has, value: verified} + actor: reviewer + - id: ready + label: Marked ready for review + fact: {fact: draft, op: "false"} + actor: reviewer + - id: confirmed + label: Stakeholder confirmed factual accuracy + fact: {fact: labels, op: has, value: confirmed} + actor: stakeholder + - id: checks-red + label: A check failed + fact: {fact: pr.checks, op: is, value: red} + - id: checks-green + label: Checks green + fact: {fact: pr.checks, op: is, value: green} + - id: merged + label: Deliverable merged + fact: {fact: merged, op: "true"} + actor: approver + - id: closed + label: Closed + fact: {fact: open, op: "false"} + - id: held + label: Hold applied (escalation) + fact: {fact: hold, op: "true"} + - id: released + label: Hold released + fact: {fact: hold, op: "false"} + actor: administrator + agents: + - profile: techdocs-drafter + states: [accepted, drafting] + tools: [read-repo, web-research, write-draft-pr] + produces: [pull-request, push] + - profile: techdocs-verifier + states: [drafting] + tools: [read-repo, web-research, comment] + produces: [comment] + checks: + - id: format-check + name: Format checks + kind: ci + required: true + source: .github/workflows/format-check.yml + - id: link-check + name: Link checks + kind: ci + required: true + source: .github/workflows/link-check.yml + - id: spell-check + name: Spelling checks + kind: ci + required: true + source: .github/workflows/spell-check.yml + - id: provenance-block + name: Provenance block present and strict check passes + kind: manual + required: true + source: "spec:15. The provenance block#L793-L877" + bookkeeping: + - on: accepted + action: notify-contacts + params: {note: "mention the validated project contacts on the intake issue before drafting (HC-6)", source: "spec:12. Lifecycle bindings#L482-L636"} + - on: ready + action: mention-stakeholders + params: {note: "mention the stakeholder set frozen at acceptance with the participant guide link", source: "spec:12. Lifecycle bindings#L482-L636"} + - on: merged + action: open-next-phase-tracking-issue + params: {note: "open the next phase's tracking issue labeled phase/* and needs-triage (phase C when skip-implementation is set); close the intake as completed after the final phase", source: "spec:12. Lifecycle bindings#L482-L636"} + labels: + entries: + - {name: Docs analysis, color: c2e0c6, description: CNCF technical documentation assistance} + - {name: needs-triage, color: EDEDED, description: Assessment request awaiting triage, state: requested} + - {name: triage/accepted, color: C2E0C6, description: Assessment request accepted, state: accepted} + - {name: triage/declined, color: F9D0C4, description: "Assessment request declined, reason posted", state: declined} + - {name: phase/assessment, color: C5DEF5, description: "Assessment phase A: analysis"} + - {name: phase/implementation, color: 84B6EB, description: "Assessment phase B: implementation plan"} + - {name: phase/backlog, color: 1D76DB, description: "Assessment phase C: issue backlog"} + - {name: verified, color: 0E8A16, description: Verification complete for this deliverable PR, state: in-review} + - {name: confirmed, color: 006B75, description: Stakeholder confirmation recorded for this deliverable PR, state: awaiting-merge} + - {name: skip-implementation, color: D4C5F9, description: Phase B skipped; phase A merge opens phase C} + transitions: + # issues + - {from: requested, to: accepted, on: accepted, gateRef: accept, note: "spec §12 Accept"} + - {from: requested, to: declined, on: declined, gateRef: decline, note: "spec §12 Accept: /decline"} + - {from: requested, to: closed, on: closed, note: "withdrawn by the project"} + - {from: accepted, to: draft-open, on: draft-opened, note: "spec §12 Draft"} + - {from: accepted, to: escalated, on: held, note: "spec §5 failure path: escalated to the administrator"} + - {from: accepted, to: closed, on: closed, note: "tracking issue declined or assessment ended"} + - {from: draft-open, to: accepted, on: draft-withdrawn, note: "draft discarded: restart, hand-write or stop"} + - {from: draft-open, to: escalated, on: held, note: "spec §5 failure path"} + - {from: draft-open, to: closed, on: closed, note: "phase merged (next phase opens) or aborted"} + - {from: escalated, to: accepted, on: released, note: "administrator released the hold"} + - {from: escalated, to: closed, on: closed, gateRef: abort, note: "spec §12 Failure and abort"} + # pull requests + - {from: drafting, to: in-review, on: verified, gateRef: verified, note: "spec §12 Review: verified label is the gate"} + - {from: drafting, to: discarded, on: closed, gateRef: discard} + - {from: in-review, to: stakeholder-review, on: ready, gateRef: ready, note: "spec §12 Stakeholder review: /ready"} + - {from: in-review, to: discarded, on: closed, gateRef: discard} + - {from: checks-failing, to: stakeholder-review, on: checks-green} + - {from: checks-failing, to: discarded, on: closed, gateRef: discard} + - {from: stakeholder-review, to: awaiting-merge, on: confirmed, gateRef: confirm, note: "spec §12: /confirm from the frozen stakeholder set"} + - {from: stakeholder-review, to: checks-failing, on: checks-red} + - {from: stakeholder-review, to: discarded, on: closed, gateRef: discard} + - {from: awaiting-merge, to: merged, on: merged, gateRef: approve, note: "spec §5 step 6 / §12 Merge"} + - {from: awaiting-merge, to: checks-failing, on: checks-red} + - {from: awaiting-merge, to: discarded, on: closed, gateRef: discard} + derivation: + mode: rules + rules: + # issues (intake umbrella and phase tracking issues) + - id: declined + kind: issue + state: declined + when: + - {fact: open, op: "false"} + - {fact: labels, op: has, value: triage/declined} + reason: declined by a technical writer (triage/declined) + - id: closed + kind: issue + state: closed + when: + - {fact: open, op: "false"} + reason: assessment ended (completed at the final merge, or aborted) + - id: escalated + kind: issue + state: escalated + when: + - {fact: hold, op: "true"} + reason: on hold; escalated to the administrator/platform owner + - id: draft-open + kind: issue + state: draft-open + when: + - {fact: labels, op: has, value: triage/accepted} + - {fact: issue.covered, op: "true"} + reason: a draft PR covers this phase + - id: accepted + kind: issue + state: accepted + when: + - {fact: labels, op: has, value: triage/accepted} + reason: accepted; awaiting a draft + - id: requested + kind: issue + state: requested + when: + - {fact: labels, op: has, value: needs-triage} + reason: filed; awaiting triage + # pull requests (the deliverable carries the Docs analysis label) + - id: merged + kind: pr + state: merged + when: + - {fact: merged, op: "true"} + - {fact: labels, op: has, value: Docs analysis} + reason: deliverable merged by the approver + - id: discarded + kind: pr + state: discarded + when: + - {fact: open, op: "false"} + - {fact: labels, op: has, value: Docs analysis} + reason: draft discarded (closed unmerged) + - id: in-review-draft + kind: pr + state: in-review + when: + - {fact: draft, op: "true"} + - {fact: labels, op: has, value: verified} + - {fact: labels, op: has, value: Docs analysis} + reason: verified; the reviewer refines and checks findings against source + - id: drafting + kind: pr + state: drafting + when: + - {fact: draft, op: "true"} + - {fact: labels, op: has, value: Docs analysis} + reason: AI-drafted deliverable in progress; verification not yet recorded + - id: checks-failing + kind: pr + state: checks-failing + when: + - {fact: pr.checks, op: is, value: red} + - {fact: labels, op: has, value: Docs analysis} + reason: a format, link or spelling check is red + - id: awaiting-merge + kind: pr + state: awaiting-merge + when: + - {fact: labels, op: has, value: confirmed} + - {fact: labels, op: has, value: Docs analysis} + reason: stakeholder confirmation recorded; awaiting the approver + - id: stakeholder-review + kind: pr + state: stakeholder-review + when: + - {fact: labels, op: has, value: verified} + - {fact: labels, op: has, value: Docs analysis} + reason: ready for review; awaiting /confirm from a stakeholder + - id: in-review-unverified + kind: pr + state: in-review + when: + - {fact: labels, op: has, value: Docs analysis} + reason: ready without the verified label; verification is still owed + pipelines: + - id: gha-github-workflows-format-check + name: Format checks + system: github-actions + path: .github/workflows/format-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: format-check, name: FILE FORMAT, kind: job, runsOn: [ubuntu-latest], level: 0} + - {id: markdown-linter, name: MARKDOWN linter, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + - id: gha-github-workflows-link-check + name: Link checks + system: github-actions + path: .github/workflows/link-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: link-check, name: LINK checking, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + - id: gha-github-workflows-spell-check + name: Spelling checks + system: github-actions + path: .github/workflows/spell-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: spelling-check, name: SPELLING check, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + coverage: + files: 3 + parsed: 3 diff --git a/src/pkg/hub/marketplace/curated/curated_test.go b/src/pkg/hub/marketplace/curated/curated_test.go index 81700c3232..975017449d 100644 --- a/src/pkg/hub/marketplace/curated/curated_test.go +++ b/src/pkg/hub/marketplace/curated/curated_test.go @@ -2,6 +2,7 @@ package curated_test import ( "errors" + "reflect" "strings" "testing" "time" @@ -52,8 +53,79 @@ func TestLoadEmbedded(t *testing.T) { t.Fatalf("%s: tier %q", e.Path, b.Tier) } } - if !seen["cncf-techdocs-assessment@curated-2026.11.1"] { - t.Fatalf("techdocs model missing: %v", seen) + for _, key := range []string{techdocsKey2026_11_1, techdocsKey2026_11_2} { + if !seen[key] { + t.Fatalf("techdocs model %s missing: %v", key, seen) + } + } +} + +const ( + techdocsKey2026_11_1 = "cncf-techdocs-assessment@curated-2026.11.1" + techdocsKey2026_11_2 = "cncf-techdocs-assessment@curated-2026.11.2" +) + +// techdocsVersion returns the embedded cncf-techdocs-assessment model at one +// spec.meta.version. +func techdocsVersion(t *testing.T, version string) *workflowmodel.Model { + t.Helper() + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + for _, e := range entries { + if e.Model.Metadata.Name == "cncf-techdocs-assessment" && e.Model.Spec.Meta.Version == version { + return e.Model + } + } + t.Fatalf("cncf-techdocs-assessment@%s not embedded", version) + return nil +} + +// TestTechDocsIntakeHook pins hivecommons/hive#18139 step 4: curated-2026.11.2 +// declares the intake hook with the rendered assessment-request.yml headings, +// curated-2026.11.1 stays as first published and declares nothing, and the +// two versions differ in nothing else. +func TestTechDocsIntakeHook(t *testing.T) { + v1 := techdocsVersion(t, "curated-2026.11.1") + v2 := techdocsVersion(t, "curated-2026.11.2") + if v1.Spec.Intake.DerivesReferenceRepos() || len(v1.Spec.Intake.OutOfScopeFields) != 0 { + t.Fatalf("2026.11.1 must declare no intake fields (never edited in place): %+v", v1.Spec.Intake) + } + if !v2.Spec.Intake.DerivesReferenceRepos() { + t.Fatalf("2026.11.2 must declare intake.referenceRepoFields: %+v", v2.Spec.Intake) + } + wantRefs := []string{"Website or documentation repository", "Other repositories or sites in scope"} + wantOut := []string{"Out of scope"} + if !reflect.DeepEqual(v2.Spec.Intake.ReferenceRepoFields, wantRefs) || !reflect.DeepEqual(v2.Spec.Intake.OutOfScopeFields, wantOut) { + t.Fatalf("2026.11.2 intake = %+v; want refs %v out %v", v2.Spec.Intake, wantRefs, wantOut) + } + if _, ok := v2.Metadata.Annotations["workflow.hivecommons.dev/changes"]; !ok { + t.Fatal("2026.11.2 must carry a workflow.hivecommons.dev/changes annotation") + } + // Everything but the version, the changes annotation and the intake block + // is identical, so a hive moving from 11.1 to 11.2 changes nothing in its + // lifecycle. + a, b := *v1, *v2 + a.Spec.Meta.Version, b.Spec.Meta.Version = "", "" + a.Spec.Intake, b.Spec.Intake = workflowmodel.Intake{}, workflowmodel.Intake{} + a.Metadata.Annotations, b.Metadata.Annotations = nil, nil + if a.Hash() != b.Hash() { + t.Fatal("2026.11.2 must differ from 2026.11.1 only in version, annotations and intake") + } + if !reflect.DeepEqual(a.Metadata, b.Metadata) { + t.Fatalf("metadata differs beyond annotations:\n%+v\n%+v", a.Metadata, b.Metadata) + } + // The rules evaluator derives the same stages from both versions. + for _, m := range []*workflowmodel.Model{v1, v2} { + p, diags, err := workflowmodel.Compile(m) + if err != nil || len(diags) != 0 { + t.Fatalf("%s: compile err=%v diags=%+v", m.Spec.Meta.Version, err, diags) + } + got := stageengine.DeriveFromModel(p, stagelabels.IssueItem).Derive(issue("triage/accepted")) + if got.Abstain || got.Stage != stagelabels.Claimed { + t.Fatalf("%s: accepted issue derived %+v", m.Spec.Meta.Version, got) + } } } @@ -82,23 +154,15 @@ func TestDecodeRejects(t *testing.T) { } } +// techdocs compiles the latest embedded cncf-techdocs-assessment version; +// TestTechDocsIntakeHook pins that every version derives the same stages. func techdocs(t *testing.T) *workflowmodel.Profile { t.Helper() - entries, errs := curated.Load() - if len(errs) != 0 { - t.Fatal(errs) - } - for _, e := range entries { - if e.Model.Metadata.Name == "cncf-techdocs-assessment" { - p, _, err := workflowmodel.Compile(e.Model) - if err != nil { - t.Fatal(err) - } - return p - } + p, _, err := workflowmodel.Compile(techdocsVersion(t, "curated-2026.11.2")) + if err != nil { + t.Fatal(err) } - t.Fatal("cncf-techdocs-assessment not embedded") - return nil + return p } func issue(labels ...string) stageengine.Facts { diff --git a/src/pkg/hub/marketplace_curated_test.go b/src/pkg/hub/marketplace_curated_test.go index b08a0786cb..5eabf79136 100644 --- a/src/pkg/hub/marketplace_curated_test.go +++ b/src/pkg/hub/marketplace_curated_test.go @@ -15,11 +15,22 @@ import ( "github.com/hivecommons/hive/pkg/workflowmodel/bundle" ) -const mktTechDocsKey = "cncf-techdocs-assessment@curated-2026.11.1" +// The embedded cncf-techdocs-assessment versions. Curated files are never +// edited in place, so each version is its own file and the catalogue keeps +// both (hivecommons/hive#18139 step 4 added 2026.11.2). +const ( + mktTechDocsName = "cncf-techdocs-assessment" + mktTechDocsVersion1 = "curated-2026.11.1" + mktTechDocsVersion2 = "curated-2026.11.2" + mktTechDocsKey = mktTechDocsName + "@" + mktTechDocsVersion1 + mktTechDocsKeyLatest = mktTechDocsName + "@" + mktTechDocsVersion2 + mktTechDocsVersionCnt = 2 +) // TestMarketplaceCuratedSeedPublishesAndIsIdempotent: the embedded curated // models land at tier curated on the first seed; a second seed publishes -// nothing and leaves exactly one version per model. +// nothing and leaves exactly the embedded versions per model, the newest +// last in the index. func TestMarketplaceCuratedSeedPublishesAndIsIdempotent(t *testing.T) { s := marketplaceTestHub(t, nil) st := s.marketplace() @@ -29,10 +40,17 @@ func TestMarketplaceCuratedSeedPublishesAndIsIdempotent(t *testing.T) { if len(errs) != 0 { t.Fatalf("seed errors: %v", errs) } - if len(published) != mktCuratedCount(t) || published[0] != mktTechDocsKey { + if len(published) != mktCuratedCount(t) { t.Fatalf("published = %v", published) } - raw, err := st.Get("cncf-techdocs-assessment", "curated-2026.11.1") + seeded := map[string]bool{} + for _, k := range published { + seeded[k] = true + } + if !seeded[mktTechDocsKey] || !seeded[mktTechDocsKeyLatest] { + t.Fatalf("published = %v; want both %s and %s", published, mktTechDocsKey, mktTechDocsKeyLatest) + } + raw, err := st.Get(mktTechDocsName, mktTechDocsVersion1) if err != nil { t.Fatal(err) } @@ -46,21 +64,40 @@ func TestMarketplaceCuratedSeedPublishesAndIsIdempotent(t *testing.T) { if b.Model.Spec.Scope.Repository != "cncf/techdocs" { t.Fatalf("scope: %+v", b.Model.Spec.Scope) } + if b.Model.Spec.Intake.DerivesReferenceRepos() { + t.Fatalf("%s must not declare intake fields: %+v", mktTechDocsKey, b.Model.Spec.Intake) + } + raw, err = st.Get(mktTechDocsName, mktTechDocsVersion2) + if err != nil { + t.Fatal(err) + } + latest, err := bundle.Decode(raw) + if err != nil { + t.Fatal(err) + } + if latest.Tier != bundle.TierCurated || !latest.Model.Spec.Intake.DerivesReferenceRepos() { + t.Fatalf("%s: tier=%s intake=%+v", mktTechDocsKeyLatest, latest.Tier, latest.Model.Spec.Intake) + } published, errs = s.seedCuratedModels(st, now.Add(time.Hour)) if len(published) != 0 || len(errs) != 0 { t.Fatalf("second seed: published=%v errs=%v", published, errs) } - vs, err := st.Versions("cncf-techdocs-assessment") - if err != nil || len(vs) != 1 { + vs, err := st.Versions(mktTechDocsName) + if err != nil || len(vs) != mktTechDocsVersionCnt { t.Fatalf("versions: %v %d", err, len(vs)) } + // The listing exposes both versions, oldest first, so the last entry is + // the one a fresh import picks up. + if vs[0].Version != mktTechDocsVersion1 || vs[len(vs)-1].Version != mktTechDocsVersion2 { + t.Fatalf("versions order: %+v", vs) + } idx, _, _, err := st.Index() if err != nil { t.Fatal(err) } for _, e := range idx.Entries { - if e.Name == "cncf-techdocs-assessment" && (e.Tier != bundle.TierCurated || e.NewerIngestAvailable) { + if e.Name == mktTechDocsName && (e.Tier != bundle.TierCurated || e.NewerIngestAvailable) { t.Fatalf("index entry: %+v", e) } } @@ -147,9 +184,9 @@ func TestMarketplaceStartRefreshSeedsCurated(t *testing.T) { done := make(chan struct{}) go func() { s.StartMarketplaceRefresh(ctx); close(done) }() testutil.Eventually(t, 5*time.Second, func() bool { - vs, err := s.marketplace().Versions("cncf-techdocs-assessment") - return err == nil && len(vs) == 1 - }, "curated model not seeded at startup") + vs, err := s.marketplace().Versions(mktTechDocsName) + return err == nil && len(vs) == mktTechDocsVersionCnt + }, "curated models not seeded at startup") cancel() select { case <-done: diff --git a/src/pkg/refderive/derive_curated_test.go b/src/pkg/refderive/derive_curated_test.go new file mode 100644 index 0000000000..e88be976c6 --- /dev/null +++ b/src/pkg/refderive/derive_curated_test.go @@ -0,0 +1,67 @@ +package refderive + +import ( + "errors" + "testing" + + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// curatedTechdocs returns the embedded cncf-techdocs-assessment model at one +// spec.meta.version, as a hive that imported it from the marketplace holds it. +func curatedTechdocs(t *testing.T, version string) *workflowmodel.Model { + t.Helper() + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + for _, e := range entries { + if e.Model.Metadata.Name == "cncf-techdocs-assessment" && e.Model.Spec.Meta.Version == version { + return e.Model + } + } + t.Fatalf("cncf-techdocs-assessment@%s not embedded", version) + return nil +} + +// TestDeriveWithEmbeddedCuratedModels pins hivecommons/hive#18139 step 4 +// against the real curated files: a hive bound to curated-2026.11.1 derives +// nothing (the version declares no intake fields and is never edited in +// place), binding curated-2026.11.2 derives the same repos the hand-written +// intake in derive_test.go does. +func TestDeriveWithEmbeddedCuratedModels(t *testing.T) { + opts := Options{Authorized: []string{"cncf/techdocs"}} + + old := curatedTechdocs(t, "curated-2026.11.1") + if old.Spec.Intake.DerivesReferenceRepos() { + t.Fatalf("curated-2026.11.1 must declare no intake fields: %+v", old.Spec.Intake) + } + if _, err := Derive(accepted395(t), old.Spec.Intake, opts); !errors.Is(err, ErrNoIntakeFields) { + t.Fatalf("curated-2026.11.1: err %v, want ErrNoIntakeFields", err) + } + + cur := curatedTechdocs(t, "curated-2026.11.2") + if cur.Spec.Intake.ReferenceRepoFields == nil || cur.Spec.Intake.OutOfScopeFields == nil { + t.Fatalf("curated-2026.11.2 intake: %+v", cur.Spec.Intake) + } + fromModel, err := Derive(accepted395(t), cur.Spec.Intake, opts) + if err != nil { + t.Fatalf("curated-2026.11.2: %v", err) + } + fromTest, err := Derive(accepted395(t), techdocsIntake, opts) + if err != nil { + t.Fatal(err) + } + if len(fromModel.Repos) == 0 || len(fromModel.Repos) != len(fromTest.Repos) { + t.Fatalf("curated-2026.11.2 derived %+v, hand-written intake derived %+v", fromModel.Repos, fromTest.Repos) + } + for i := range fromModel.Repos { + if fromModel.Repos[i].Repo != fromTest.Repos[i].Repo || fromModel.Repos[i].DerivedFrom != "cncf/techdocs#395" { + t.Fatalf("repo %d: model %+v, test %+v", i, fromModel.Repos[i], fromTest.Repos[i]) + } + } + if len(fromModel.Skipped) != 0 { + t.Fatalf("nothing in #395 should be skipped: %+v", fromModel.Skipped) + } +} From 8b1f9639c332f2acb6be5099511e06742be7fe54 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 02:30:14 +0000 Subject: [PATCH 202/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.252.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.252.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-18146-selfservice-expiry-and-type.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-18146-selfservice-expiry-and-type.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 45591f974b..d5c6303814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.252.0) + +### Added + +- Hub: self-service (`byo-`) hives now expire. A background sweep (every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL`, default 10m) revokes the key and removes the registry entry of any registration that went longer than `HIVE_HUB_SELFSERVICE_EXPIRY` (default 24h) without an accepted heartbeat, writing `audit: selfservice_hive_expired`. The hub also assigns every `byo-` registry entry the hive type `self-hosted` from its ID prefix, ignoring any `hive_type` the heartbeat claims, so the dashboard and `/api/hub/*` listings can badge them (#18146, part of #18060). + ## 2026-11-14 (v5.251.0) ### Added diff --git a/changelog.d/added-18146-selfservice-expiry-and-type.md b/changelog.d/added-18146-selfservice-expiry-and-type.md deleted file mode 100644 index bc8cd7fba8..0000000000 --- a/changelog.d/added-18146-selfservice-expiry-and-type.md +++ /dev/null @@ -1 +0,0 @@ -- Hub: self-service (`byo-`) hives now expire. A background sweep (every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL`, default 10m) revokes the key and removes the registry entry of any registration that went longer than `HIVE_HUB_SELFSERVICE_EXPIRY` (default 24h) without an accepted heartbeat, writing `audit: selfservice_hive_expired`. The hub also assigns every `byo-` registry entry the hive type `self-hosted` from its ID prefix, ignoring any `hive_type` the heartbeat claims, so the dashboard and `/api/hub/*` listings can badge them (#18146, part of #18060). From b1f1140347028f1b85512504d59d18a9bb777668 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 21:39:31 -0500 Subject: [PATCH 203/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20self-service=20h?= =?UTF-8?q?ive=20registration=20flow=20in=20get-started,=20hub-deployment,?= =?UTF-8?q?=20env-vars=20and=20owner=20guide=20(#18172)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: self-service hive registration in get-started, hub-deployment, env-vars and a new owner guide The /get-started#self-host card no longer tells self-hosters to open an issue or obtain HIVE_HUB_SECRET from the operator. It describes the flow that shipped in #18150, #18155, #18167 and #18168: sign in, Register your own hive, admin on each repo, copy the key once, paste the Compose, Podman or Kubernetes snippet, rotate or remove from Self-hosted hives, 24h expiry. hub-deployment.md gains a Self-service registration section for operators (flags with defaults, what a registration does, what a byo- key can and cannot do, rotation overlap, sweep, admin revoke, audit events). env-vars.md amends HIVE_HUB_SECRET and HIVE_HEARTBEAT_KEY for byo- hives. register-self-hosted-hive.md is the owner-facing page, linked from README. Fixes #18147 Part of #18060 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 📖 get-started: drop inline style on the self-host heading to keep the style ratchet at baseline Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ged-18147-selfservice-registration-docs.md | 1 + src/docs/README.md | 2 +- src/docs/env-vars.md | 4 +- src/docs/hub-deployment.md | 58 ++++++++++++ src/docs/register-self-hosted-hive.md | 90 +++++++++++++++++++ src/pkg/hub/static/get-started.html | 12 ++- 6 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 changelog.d/changed-18147-selfservice-registration-docs.md create mode 100644 src/docs/register-self-hosted-hive.md diff --git a/changelog.d/changed-18147-selfservice-registration-docs.md b/changelog.d/changed-18147-selfservice-registration-docs.md new file mode 100644 index 0000000000..6e86ff7232 --- /dev/null +++ b/changelog.d/changed-18147-selfservice-registration-docs.md @@ -0,0 +1 @@ +- The hub's `/get-started#self-host` card now describes self-service registration (sign in, Register your own hive, copy the one-time key, paste the Compose, Podman or Kubernetes snippet) instead of telling self-hosters to open an issue and obtain `HIVE_HUB_SECRET` from the operator. `hub-deployment.md` gains a Self-service registration section for operators, `env-vars.md` notes that `byo-` hives hold only `HIVE_HEARTBEAT_KEY`, and a new `register-self-hosted-hive.md` walks owners through the flow ([#18147](https://github.com/hivecommons/hive/issues/18147), part of [#18060](https://github.com/hivecommons/hive/issues/18060)). diff --git a/src/docs/README.md b/src/docs/README.md index 4f2c046d60..357ed3217a 100644 --- a/src/docs/README.md +++ b/src/docs/README.md @@ -13,7 +13,7 @@ Choose how to obtain a running hive **before** configuring its agents: | Your situation | Start here | Expected result | |---|---|---| | You want a hosted hive, without running a cluster | [Hosted onboarding](hosted-hub.md); request at [hive.hivecommons.dev/get-started](https://hive.hivecommons.dev/get-started) | After admin approval and provisioning, **My Hives** links to your dashboard. | -| You want to run one hive yourself | [Docker Compose](quickstart-docker-compose.md), [Podman](quickstart-podman.md), or [Kubernetes](kubernetes-deployment.md) | Your own running container and dashboard, with storage and authentication configured. | +| You want to run one hive yourself | [Docker Compose](quickstart-docker-compose.md), [Podman](quickstart-podman.md), or [Kubernetes](kubernetes-deployment.md) | Your own running container and dashboard, with storage and authentication configured. To list it on a hub afterwards, see [Register a self-hosted hive](register-self-hosted-hive.md). | | Someone has already given you dashboard access | [Getting started](getting-started.md#step-0--before-you-start-do-this-first) | Connect the GitHub App, authenticate a model backend, set a budget, then enable agents conservatively. | You need a GitHub identity and a repository on GitHub.com or GitHub Enterprise. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 64425ef5f7..d1d4bfe9f8 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -83,7 +83,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_HUB_KIOSK_CTA_URL` | No | `https://hive.hivecommons.dev/get-started` (`kioskcta.DefaultURL`, `src/pkg/webassets/kioskcta`) | Destination of the call-to-action badge the hub Fleet activity page shows bottom-right when loaded with `?kiosk=1` (the fleet Twitch stream). Rendered as text and as a QR code. | | `HIVE_HUB_KIOSK_CTA_TEXT` | No | `🐝 Want this for your repos?` (`kioskcta.DefaultText`) | Copy line above the URL on that badge. | | `HIVE_HUB_LEGACY_COOKIE_DOMAIN` | No | none | Optional transition-only hub SSO cookie domain to expire alongside the active cookie domain while accepting any carried `hive_hub_user` cookie value during a host migration. Writes still target the domain derived from `HIVE_HUB_PUBLIC_URL`; unset disables the extra legacy-domain cleanup. | -| `HIVE_HUB_SECRET` | Required for spokes registered to a protected hub; optional for a standalone hub with `/data/saas/hub-secret.key` | `/data/saas/hub-secret.key` on the hub when present; no fallback for spoke heartbeat auth | Bearer secret for spoke heartbeats and hub/spoke SaaS APIs. | +| `HIVE_HUB_SECRET` | Required for spokes registered to a protected hub; optional for a standalone hub with `/data/saas/hub-secret.key` | `/data/saas/hub-secret.key` on the hub when present; no fallback for spoke heartbeat auth | Bearer secret for spoke heartbeats and hub/spoke SaaS APIs. Self-hosted (`byo-`) hives registered through self-service never receive it: they hold only their own `HIVE_HEARTBEAT_KEY`, and the hub never tries `HIVE_HUB_SECRET` or anything derived from it for a `byo-` ID. | | `HIVE_HUB_SELFSERVICE_REGISTRATION` | No | off | Enables the self-service registration API (`/api/hub/selfservice/*`) that lets a signed-in user register a hive they run themselves under the reserved `byo-<login>--<name>` ID namespace, after proving `permissions.admin` on every listed repo with their own GitHub token ([#18060](https://github.com/hivecommons/hive/issues/18060)). Accepts `1/true/yes/on`. When off, every route but `GET /api/hub/selfservice/status` answers `404` and the status reports `enabled: false`. Read per request; no restart needed. | | `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | No | `5` | Maximum self-service hives one user may have registered at once; the next registration answers `429` until one is removed. | | `HIVE_HUB_SELFSERVICE_EXPIRY` | No | `24h` | How long a self-service hive may go without an accepted heartbeat before the expiry sweep revokes its key and removes its registry entry (counted from the last accepted heartbeat, or from registration while there has been none). Reported by `GET /api/hub/selfservice/status` as `expiry`. Go duration syntax. | @@ -479,7 +479,7 @@ only when neither is configured. | Variable | Required | Default | Purpose | |---|---:|---|---| -| `HIVE_HEARTBEAT_KEY` | No | derived from `HIVE_HUB_SECRET` | Spoke heartbeat signing sub-key. | +| `HIVE_HEARTBEAT_KEY` | No | derived from `HIVE_HUB_SECRET` | Spoke heartbeat signing sub-key. For a self-service (`byo-`) hive this is the only key the spoke holds: the hub mints it at registration, shows it once, and stores only a hash. There is no derivation fallback for a `byo-` ID; rotate it from the dashboard if it is lost (see [hub-deployment.md](hub-deployment.md#self-service-registration)). | | `HIVE_SESSION_KEY` | No | derived from `HIVE_HUB_SECRET` | Spoke session-cookie signing sub-key. | | `HIVE_INVITE_KEY` | No | derived from `HIVE_HUB_SECRET` | Per-hive contributor-invite signing key. Symmetric: the spoke both mints and verifies invite tokens with it. | | `HIVE_TERMINAL_KEY` | No | self-derived per-hive from `HIVE_HUB_SECRET` + `HIVE_ID` | Per-hive terminal-assertion signing key. It never falls back to a fleet-uniform key. | diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index fb132e303f..cf77bd3989 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -92,6 +92,64 @@ Without `HIVE_HUB_SECRET`, a protected hub returns `401 unauthorized`, the spoke Hive IDs that start with `byo-` are reserved for hives that their owners run and register themselves ([#18060](https://github.com/hivecommons/hive/issues/18060)). A hub lists such a hive as directory metadata only, so its heartbeat response never carries hub-held credentials or management actions, whatever key the spoke authenticated with. The hub withholds these fields: the GitHub App config (including cluster, secondary and webhook relay keys), the pending gateway, the claimed project config, the cluster ID, the authorized-user list, visibility, upgrade and branch-switch instructions, the upgrade announcement and policy, spoke restarts, agent-restart resets, fleet-breaker instructions, and self-experiment flags. Display-only data such as admin banners still flows. When the fence withholds a field, the hub logs `heartbeat: credential fence withheld fields from self-service hive` with the field names, never the values. Do not give an operator-provisioned hive a `byo-` ID. +### Self-service registration + +The fence above is always on. The registration flow that fills the `byo-` namespace is opt-in. With it enabled, a signed-in hub user lists a hive they run on their own infrastructure without an operator issuing anything. The owner-facing steps are in [Register a self-hosted hive on a hub](register-self-hosted-hive.md); this section is for the operator running the hub. + +#### Enable it + +| Variable | Default | What it controls | +|---|---|---| +| `HIVE_HUB_SELFSERVICE_REGISTRATION` | off | Turns on `/api/hub/selfservice/*` and the **Register your own hive** form on the dashboard. Accepts `1/true/yes/on`. Read per request, so no restart is needed. While off, every route except `GET /api/hub/selfservice/status` answers `404`, and the dashboard button sends users to the self-hosting guide instead of the form. | +| `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | `5` | How many `byo-` hives one GitHub login may hold at once. The next registration answers `429` until the user removes one. | +| `HIVE_HUB_SELFSERVICE_EXPIRY` | `24h` | How long a registered hive may go without an accepted heartbeat before the sweep removes it. Counted from the last accepted heartbeat, or from registration while there has been none. | +| `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` | `10m` | How often the expiry sweep runs. Read at startup; restart after changing. | + +All four are listed with the other hub variables in [env-vars.md](env-vars.md). + +#### What a registration does + +1. The user is signed in through the hub's GitHub OAuth, so the hub holds their encrypted GitHub token. Without a stored token or a GitHub login the request is refused. +2. The user supplies a short name, an org, up to 20 `owner/name` repos and an `https` dashboard URL. The hub builds the ID as `byo-<github login>-<name>`. Nothing the user sends can produce an ID outside the `byo-` prefix, and a name that would borrow another namespace (such as `hosted-`) is refused. +3. For every repo the hub calls GitHub with the user's own token and requires `permissions.admin`. `maintain` and `write` are not enough. One failing repo refuses the whole request with `403` naming that repo, and nothing is stored. +4. The hub checks the per-user cap and that the ID is not already taken, in the registry or by a hub-provisioned hive. +5. The hub mints a random heartbeat key, stores only its hash in `/data/hub-selfservice.json`, and returns the raw key in the `201` response together with Compose, Podman and Kubernetes snippets. The raw key appears nowhere else: not in the list route, not in a log line, not on disk. + +The registry entry becomes a `self-hosted` hive in Active Hives and the fleet views once a heartbeat is accepted. The type is assigned by the hub; the spoke cannot claim it. + +#### What a `byo-` key can and cannot do + +- It authenticates heartbeats for exactly one hive ID, the one it was minted for. Presenting it with any other ID fails. +- A `byo-` ID is verified only against its stored hash. The master-derived paths are never tried for a `byo-` ID, so `HIVE_HUB_SECRET` and keys derived from it never authenticate a self-service hive, and a self-service key never authenticates a `hosted-` or operator-provisioned ID. +- The owner never receives `HIVE_HUB_SECRET`. The hub never consults it in the self-service routes and never returns it. +- Everything the fence withholds still applies: the hub sends the hive no GitHub App config, gateway, cluster ID, authorized users, upgrade or branch-switch instructions, restarts or fleet-breaker actions. Listing is directory metadata only. + +#### Rotation + +The owner can rotate from the **Self-hosted hives** list or with `POST /api/hub/selfservice/hives/{id}/rotate`. The hub mints a new key and keeps the previous hash live for a 10 minute overlap so a running spoke can pick up the new value without a gap. After the overlap only the new key is accepted. Only the owner can rotate; a hub admin who does not own the hive gets `404`, the same answer as for a missing hive. + +#### Expiry sweep + +A background sweep runs every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` and removes any `byo-` hive whose last accepted heartbeat is older than `HIVE_HUB_SELFSERVICE_EXPIRY`, or that never heartbeated within that window after registration. The key is revoked and the registry entry is removed in the same pass. The owner registers again when the hive is back; there is no grace list. + +#### Revoking as admin + +A hub admin removes any self-service hive with `DELETE /api/hub/selfservice/hives/{id}` or from the dashboard. The key stops authenticating immediately and the listing disappears. The owner's own **Remove** button does the same for their hives. The audit line records who did it and whether it was the owner or an admin. If you need to stop all registrations at once, unset `HIVE_HUB_SELFSERVICE_REGISTRATION`; existing entries keep heartbeating until they expire or are removed. + +#### Audit events + +Every decision writes one structured log line, documented in [audit-log.md](audit-log.md): + +| Event | When | +|---|---| +| `selfservice_hive_registered` | A registration passed every check and a key was minted. | +| `selfservice_hive_refused` | A registration was turned away; `reason` is one of `invalid_name`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `per_user_cap`, `no_github_token`, `no_github_login`, `not_repo_admin`, `ownership_check_error`. | +| `selfservice_hive_rotated` | The owner rotated the key; `overlap` is how long the old key keeps working. | +| `selfservice_hive_revoked` | The owner or an admin removed the hive; `by_admin` tells which. | +| `selfservice_hive_expired` | The sweep removed a hive; `last_heartbeat` is RFC 3339 or `never`. | + +No audit line ever carries a key or a hash. + ## Hosted spoke provisioning model The hub's SaaS provisioner creates or records: diff --git a/src/docs/register-self-hosted-hive.md b/src/docs/register-self-hosted-hive.md new file mode 100644 index 0000000000..afae153af7 --- /dev/null +++ b/src/docs/register-self-hosted-hive.md @@ -0,0 +1,90 @@ +# Register a self-hosted hive on a hub + +This page is for people who run their own hive and want it listed on a hub such as hive.hivecommons.dev. It mirrors the **Self-host** card on the hub's `/get-started` page. Operators who run a hub and want to enable or audit this flow should read [hub-deployment.md](hub-deployment.md#self-service-registration) instead. + +A self-hosted hive is complete on its own. Listing it on a hub is optional and gives you a public entry in Active Hives, an online badge, and a place in the fleet views. The hub lists your hive as directory metadata only. It never sends your hive credentials, upgrades or management actions, and you never receive the hub's master secret. + +## Before you start + +- A running or ready-to-run hive. See [Docker Compose](quickstart-docker-compose.md), [Podman](quickstart-podman.md) or [Kubernetes](kubernetes-deployment.md). +- A GitHub account with **admin** permission on every repo the hive will work on. The hub checks this with your own GitHub token at registration time. `maintain` or `write` is not enough. +- A hub whose operator has enabled self-service registration. If the **Register your own hive** button sends you to the self-hosting guide instead of opening a form, the hub has it off; ask that hub's operator. + +## Steps + +1. Sign in to the hub with GitHub and open **My Hives**. +2. Click **Register your own hive**. +3. Fill in a short name, the GitHub org, the repos (as `owner/name`), and the `https` URL of your hive's dashboard. Your hive ID becomes `byo-<your login>-<name>`. +4. Submit. The hub checks admin permission on each repo. One failing repo refuses the whole request and stores nothing. +5. **Copy the key now.** The dialog shows `HIVE_HEARTBEAT_KEY` exactly once and the hub keeps only a hash. Closing the dialog is final. If you miss it, rotate the key (below) rather than registering again. +6. Start the spoke with one of the snippets below, substituting the values from the dialog. +7. After the first accepted heartbeat the hive shows as online in **Self-hosted hives** and appears in Active Hives with the **self-hosted** type. + +## Snippets + +The dialog gives you these three, already filled in. Each sets the same three variables: `HIVE_HUB_URL`, `HIVE_ID` and `HIVE_HEARTBEAT_KEY`. Do not set `HIVE_HUB_SECRET`; a self-service hive never has one. + +Docker Compose: + +```yaml +services: + hive: + image: ghcr.io/hivecommons/hive:stable + environment: + HIVE_HUB_URL: "https://hive.example.com" + HIVE_ID: "byo-yourlogin-yourname" + HIVE_HEARTBEAT_KEY: "<the key shown once>" +``` + +Podman: + +```sh +podman run -d --name hive \ + -e HIVE_HUB_URL="https://hive.example.com" \ + -e HIVE_ID="byo-yourlogin-yourname" \ + -e HIVE_HEARTBEAT_KEY="<the key shown once>" \ + ghcr.io/hivecommons/hive:stable +``` + +Kubernetes (mount or `envFrom` this Secret into the spoke Deployment): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: hive-hub-credentials +type: Opaque +stringData: + HIVE_HUB_URL: "https://hive.example.com" + HIVE_ID: "byo-yourlogin-yourname" + HIVE_HEARTBEAT_KEY: "<the key shown once>" +``` + +The snippets show only the hub-facing variables. Your hive still needs its usual configuration (GitHub token or App, model backend, storage) from the quickstart you followed. + +## Rotate the key + +Open **Self-hosted hives** and click **Rotate key** on the entry. The hub mints a new key and shows it once, the same way as at registration. The previous key keeps working for ten minutes so a running spoke can be updated without a gap; after that only the new key is accepted. Rotate if the key was lost, pasted somewhere it should not have been, or if you suspect it leaked. + +## Remove the hive + +Click **Remove** on the entry. The key stops working at once and the listing disappears. Your hive keeps running; only the hub entry is gone. A hub admin can also remove any self-service hive. + +## Expiry + +A registered hive that sends no accepted heartbeat for 24 hours (the hub's default; the dialog shows the actual window) is removed automatically, key and listing together. This also applies to a registration that never heartbeated. If your hive was down for longer than that, register it again and start it with the new key. + +## Limits and what the key does + +- One person may hold a limited number of self-service hives at once; the hub's default is 5. Remove one to register another. +- The key authenticates heartbeats for your one hive ID and nothing else. It cannot act as another hive, as a hosted hive, or as the hub. +- The hub never gives a self-service hive GitHub App credentials, a gateway, a cluster ID, authorized-user lists, upgrade or branch-switch instructions, restarts or fleet-breaker actions. Display-only data such as hub banners still arrives. +- Every registration, refusal, rotation, removal and expiry is written to the hub's audit log without the key. + +## Troubleshooting + +- **The button goes to the self-hosting guide instead of a form.** The hub has self-service registration off. Ask the operator to set `HIVE_HUB_SELFSERVICE_REGISTRATION`. +- **403 naming a repo.** You do not have admin permission on that repo under the GitHub account you signed in with. Fix the permission or remove the repo from the list. +- **429.** You are at the per-user cap. Remove an entry you no longer use. +- **The hive never shows as online.** Check that all three variables are set on the spoke and that `HIVE_ID` matches the registered ID exactly. The spoke logs `hub heartbeat rejected status=401` when the key does not match; rotate and paste the new key. +- **The entry disappeared.** It expired after the hub's heartbeat window. Register again. diff --git a/src/pkg/hub/static/get-started.html b/src/pkg/hub/static/get-started.html index 7477df1c75..c2ef793c63 100644 --- a/src/pkg/hub/static/get-started.html +++ b/src/pkg/hub/static/get-started.html @@ -453,7 +453,17 @@ <h2>Your agents, your infrastructure <span class="badge badge-ready">Ready</span # Open the dashboard open http://localhost:3001</code></pre> <p>A self-hosted hive is fully functional on its own — it does not need to be attached to hive.hivecommons.dev.</p> - <p style="margin-bottom:var(--sp-0)">Listing a hive you host yourself in the <a class="text-info" href="/#hives">Active Hives</a> table is a separate, operator-mediated step. Setting <code>HIVE_HUB_URL</code> on its own is <strong>not</strong> sufficient: the hub authenticates every heartbeat against <code>HIVE_HUB_SECRET</code>, which is issued by the hub operator — unauthenticated registration was deliberately removed to prevent registry pollution. To get a self-hosted hive listed, <a class="text-info" href="https://github.com/hivecommons/hive/issues/new">open an issue</a>. If you would rather not run the infrastructure at all, use the hosted-hive wizard at the top of this page and the hub will provision and list one for you.</p> + <h3>List your hive on this hub</h3> + <p>You can list a hive you run yourself in the <a class="text-info" href="/#hives">Active Hives</a> table without asking anyone. Setting <code>HIVE_HUB_URL</code> on its own is <strong>not</strong> sufficient: the hub authenticates every heartbeat, so your hive needs a key the hub knows. You register for that key yourself:</p> + <ol> + <li><a class="text-info" href="/login">Sign in</a> to the hub with GitHub and open <strong>My Hives</strong>.</li> + <li>Click <strong>Register your own hive</strong>. Give the hive a short name, the GitHub org and the repos it will work on. You must have <strong>admin</strong> permission on each repo; the hub checks this with your own GitHub token and stores nothing until every repo passes.</li> + <li><strong>Copy the key once.</strong> The hub shows <code>HIVE_HEARTBEAT_KEY</code> exactly one time and keeps only a hash. Use the Docker Compose, Podman or Kubernetes snippet from the same dialog, or set <code>HIVE_HUB_URL</code>, <code>HIVE_ID</code> and <code>HIVE_HEARTBEAT_KEY</code> on your spoke yourself.</li> + <li>Start the spoke. After its first accepted heartbeat it appears in Active Hives with the <strong>self-hosted</strong> type and in your <strong>Self-hosted hives</strong> list.</li> + </ol> + <p>Lost the key? Use <strong>Rotate key</strong> in Self-hosted hives; the old key keeps working for ten minutes so you can update a running spoke without a gap. <strong>Remove</strong> revokes the key and delists the hive at once. A hive that sends no accepted heartbeat for 24 hours is delisted automatically; register it again when it is back.</p> + <p>Your hive ID is <code>byo-<your login>-<name></code>. The hub lists it as directory metadata only: it never sends your hive credentials, upgrades or management actions, and you never receive <code>HIVE_HUB_SECRET</code>. The key you hold authenticates that one hive ID and nothing else.</p> + <p style="margin-bottom:var(--sp-0)">A hub operator turns this on with <code>HIVE_HUB_SELFSERVICE_REGISTRATION</code>. On a hub where it is off, the Register your own hive button brings you here instead of opening the form; ask that hub's operator to enable it. If you would rather not run the infrastructure at all, use the hosted-hive wizard at the top of this page and the hub will provision and list one for you.</p> </div> </section> From 3381487466fb6fcb3d7ff3eee319f8b88b4ceec6 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 21:47:45 -0500 Subject: [PATCH 204/223] =?UTF-8?q?=E2=9C=A8=20reference=20repos:=20derive?= =?UTF-8?q?=20write-tier=20entries=20from=20accepted=20request=20issues=20?= =?UTF-8?q?at=20load=20and=20on=20every=20reload,=20in=20memory=20only=20(?= =?UTF-8?q?#18139=20step=203)=20(#18174)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ reference repos: derive write-tier entries from accepted request issues at load and on every reload, in memory only (#18139 step 3) For each authorized repo whose bound workflow model declares intake.referenceRepoFields, list open triage/accepted issues, read the label applier from the timeline and their repo permission, run refderive.Derive and merge the result under the hand-written project.reference_repos (hand-written wins, authorized repos never derived, first duplicate wins, write clamped to issues+prs). The merged list is installed on the proxy at boot, re-merged and re-listed on every config reload, and re-listed on HIVE_REFERENCE_REPOS_DERIVE_INTERVAL (default 10m) so a closed issue or removed label drops the entry, its policy and its kick line. Derived entries never reach a saved config; the kick's REFERENCE REPOS block shows (from owner/repo#N) and the read-only reason. Fixes #18159 Part of #18139 Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 reference repos: drop derived entries from every persisted config copy yaml:"-" hides the provenance field but the derived entry itself still marshalled as a plain repo; redactedForPersist now strips derived entries on both the runtime config and dashboard overlay save paths (#18159). Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...ed-18159-derive-reference-repos-on-load.md | 1 + src/cmd/hive/boot.go | 4 + src/cmd/hive/main.go | 4 + src/cmd/hive/reference_repos.go | 90 ++-- src/cmd/hive/reference_repos_derive.go | 447 ++++++++++++++++++ src/cmd/hive/reference_repos_derive_test.go | 422 +++++++++++++++++ src/crux/proxy/reference_repos.go | 11 + src/docs/env-vars.md | 1 + src/pkg/config/config.go | 6 + src/pkg/config/reference_repos.go | 30 ++ src/pkg/config/reference_repos_test.go | 33 ++ src/pkg/github/reference_repo_request.go | 149 ++++++ src/pkg/github/reference_repo_request_test.go | 97 ++++ src/pkg/scheduler/agent_repo_scope_test.go | 25 + src/pkg/scheduler/kickmessage.go | 24 +- 15 files changed, 1312 insertions(+), 32 deletions(-) create mode 100644 changelog.d/added-18159-derive-reference-repos-on-load.md create mode 100644 src/cmd/hive/reference_repos_derive.go create mode 100644 src/cmd/hive/reference_repos_derive_test.go diff --git a/changelog.d/added-18159-derive-reference-repos-on-load.md b/changelog.d/added-18159-derive-reference-repos-on-load.md new file mode 100644 index 0000000000..6f1ad327b3 --- /dev/null +++ b/changelog.d/added-18159-derive-reference-repos-on-load.md @@ -0,0 +1 @@ +- `project.reference_repos` entries are now derived from accepted request issues at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`), in memory only: for each authorized repo whose bound workflow model declares `intake.referenceRepoFields`, the hive lists open issues carrying `triage/accepted`, reads who applied the label from the timeline and their repository permission, runs the request-form parser and installs the result on the proxy at the `issues+prs` tier (hand-written entries win on the same repo, authorized repos are never derived, duplicates keep the first). Closing the request issue or removing the label drops the entry, its proxy policy and its kick line on the next pass; a saved overlay or runtime config never contains a derived entry; the kick's REFERENCE REPOS block appends `(from owner/repo#N)` and, for a read-only derived entry, the reason (#18139, #18159). diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 93a38730a6..5ae0b55221 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -246,6 +246,10 @@ type boot struct { // bootProxy githubProxy *proxy.GitHubProxy + // referenceRepos keeps the derived reference-repo state between the + // boot pass, config reloads and the periodic re-derive + // (hivecommons/hive#18159); nil until wireReferenceRepos ran. + referenceRepos *referenceRepoRuntime // inferenceSink is the one per-agent inference usage ledger; the proxy // records into it in-process and the hive callback's usage capability // records into it for a sidecar runtime (#14138 stage 4). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index f1ec32f26d..32f602b168 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3878,6 +3878,10 @@ func (b *boot) bootWatchersWith(deps bootWatchersDeps) { // Re-sync subsystems that cache config values b.ghClient.SetRepos(b.cfg.Project.Repos) + // project.reference_repos: the file carries only the hand-written + // entries, so the derived ones are merged back now and re-listed + // from the request issues in the background (hivecommons/hive#18159). + b.reconcileReferenceReposAfterReload() b.observeWritebackRepos() if b.metricsCollector != nil { b.metricsCollector.SetProjectScope(b.cfg.Project.Org, metricsPrimaryRepo(b.cfg.Project), b.cfg.Project.Repos, b.cfg.EffectiveAIAuthor()) diff --git a/src/cmd/hive/reference_repos.go b/src/cmd/hive/reference_repos.go index a812de7adb..cec6d5f0c3 100644 --- a/src/cmd/hive/reference_repos.go +++ b/src/cmd/hive/reference_repos.go @@ -11,6 +11,7 @@ import ( "github.com/hivecommons/hive/crux/proxy" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" ) // referenceRepoCredentialExitCode marks an exit caused by a @@ -133,46 +134,73 @@ func notifyDerivedReadOnly(ctx context.Context, repos []config.ReferenceRepo, po // wireReferenceRepos installs project.reference_repos on the proxy, refusing // to boot when a hand-written write-mode entry has no usable credential. A // derived entry never refuses boot: it is downgraded to read-only and the -// request issue is told once (hivecommons/hive#18158). +// request issue is told once (hivecommons/hive#18158). With a GitHub client +// the derived entries are listed from accepted request issues first +// (hivecommons/hive#18159) and kept current by every config reload and a +// periodic pass. func (b *boot) wireReferenceRepos() { - refs := b.cfg.Project.ReferenceRepos - if len(refs) == 0 || b.githubProxy == nil { + if b.githubProxy == nil { return } - deps := referenceRepoDeps{auth: func() *github.AppAuth { return nil }} - var post derivedReadOnlyPoster - if b.ghClient != nil { - post = b.ghClient.CommentDerivedReadOnly - if auth := b.ghClient.AppAuth(); auth != nil { - apiURL := auth.APIURL() - deps.botLogin = auth.BotLogin() - deps.discover = auth.DiscoverInstallationID - deps.whoAmI = func(ctx context.Context, token string) (string, error) { - return github.TokenUserLogin(ctx, token, apiURL) - } - // Re-fetched per call: the AppAuth is replaced on key rotation. - deps.auth = func() *github.AppAuth { return b.ghClient.AppAuth() } - } else { - deps.whoAmI = func(ctx context.Context, token string) (string, error) { - return github.TokenUserLogin(ctx, token, "") - } + rt := b.newReferenceRepoRuntime() + if len(b.cfg.Project.ReferenceRepos) == 0 && rt.deriver == nil { + return + } + b.wireReferenceReposWith(rt) +} + +// newReferenceRepoRuntime builds the seams from the live GitHub client. The +// closures read b.ghClient on every call because the client is replaced on +// App key rotation. +func (b *boot) newReferenceRepoRuntime() *referenceRepoRuntime { + rt := &referenceRepoRuntime{ + deps: referenceRepoDeps{auth: func() *github.AppAuth { return nil }}, + interval: referenceRepoDeriveInterval(), + } + if b.ghClient == nil { + return rt + } + rt.post = func(ctx context.Context, issueRepo string, number int, readOnly []config.ReferenceRepo) (bool, error) { + return b.ghClient.CommentDerivedReadOnly(ctx, issueRepo, number, readOnly) + } + if auth := b.ghClient.AppAuth(); auth != nil { + apiURL := auth.APIURL() + rt.deps.botLogin = auth.BotLogin() + rt.deps.discover = auth.DiscoverInstallationID + rt.deps.whoAmI = func(ctx context.Context, token string) (string, error) { + return github.TokenUserLogin(ctx, token, apiURL) + } + // Re-fetched per call: the AppAuth is replaced on key rotation. + rt.deps.auth = func() *github.AppAuth { return b.ghClient.AppAuth() } + } else { + rt.deps.whoAmI = func(ctx context.Context, token string) (string, error) { + return github.TokenUserLogin(ctx, token, "") } } - res, err := buildReferenceRepoPolicies(b.ctx, refs, deps, b.logger) - if err != nil { + rt.deriver = newReferenceRepoDeriver( + func(ctx context.Context, repo, label string, limit int) ([]github.RequestIssueCandidate, error) { + return b.ghClient.ListRequestIssueCandidates(ctx, repo, label, limit) + }, + func(ctx context.Context, repo string, number int, label string) (github.RequestIssueAccepter, error) { + return b.ghClient.RequestIssueAccepterFor(ctx, repo, number, label) + }, + workflowIntakeResolver(b.cfg, wfstore.Open(wfstore.Root(b.cfg.Data.WorkflowsDir))), + b.cfg.GitHub.HostLabel(), + ) + return rt +} + +// wireReferenceReposWith is wireReferenceRepos with the runtime injected +// (tests supply fake listers). It runs the boot pass, exits on a hand-written +// credential error and starts the periodic re-derive. +func (b *boot) wireReferenceReposWith(rt *referenceRepoRuntime) { + b.referenceRepos = rt + if err := b.applyReferenceRepos(b.ctx, rt); err != nil { b.logger.Error("refusing to start: project.reference_repos write-mode entry has no usable credential (#18015)", "error", err.Error()) if errors.Is(err, github.ErrReferenceRepoNoCredential) { fmt.Fprintln(os.Stderr, err.Error()) } os.Exit(referenceRepoCredentialExitCode) } - // The resolved entries replace the list in memory only: a derived entry - // is never written back to any config file, and its credential state is - // what the kick and /api/status read. - b.cfg.Project.ReferenceRepos = res.repos - b.githubProxy.SetReferenceRepos(res.policies) - b.logger.Info("reference repos wired", "count", len(res.policies), "repos", b.cfg.ReferenceRepoSlugs()) - // Off the boot path: the notice is a GitHub write and must not delay - // the proxy coming up. - go notifyDerivedReadOnly(b.ctx, res.repos, post, b.logger) + b.startReferenceRepoDerivePoller(b.ctx, rt) } diff --git a/src/cmd/hive/reference_repos_derive.go b/src/cmd/hive/reference_repos_derive.go new file mode 100644 index 0000000000..f4914fa75c --- /dev/null +++ b/src/cmd/hive/reference_repos_derive.go @@ -0,0 +1,447 @@ +package main + +import ( + "context" + "log/slog" + "os" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/refderive" + "github.com/hivecommons/hive/pkg/workflowmodel" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// Derived reference repos (hivecommons/hive#18159, RFC #18139 step 3). +// +// An assessment hive's authorized repo holds request issues. Once a +// maintainer applies refderive.GrantLabel to one, the repos the request names +// become project.reference_repos entries at the issues+prs tier, in memory +// only: this file lists those issues at boot, on every config reload and on +// a timer, runs the pure refderive.Derive on each, merges the result under +// the operator's hand-written entries and installs the outcome on the proxy. +// Nothing here writes a config file; a closed issue or a removed label simply +// stops yielding entries and the next pass drops them. + +// referenceRepoDeriveIntervalEnv overrides how often the hive re-lists the +// request issues between config reloads. +const referenceRepoDeriveIntervalEnv = "HIVE_REFERENCE_REPOS_DERIVE_INTERVAL" + +// referenceRepoDeriveIntervalDefault is the re-derive cadence when the env +// var is unset or unusable: long enough to cost one list call per authorized +// repo per ten minutes, short enough that an ended grant is gone before an +// agent's next session. +const referenceRepoDeriveIntervalDefault = 10 * time.Minute + +// maxRequestIssuesPerRepoPerPass bounds how many labelled issues one pass +// reads per authorized repo: an assessment hive has a handful of open +// requests, and a repo with hundreds of triage/accepted issues is not one +// whose model should be deriving anything. +const maxRequestIssuesPerRepoPerPass = 50 + +// referenceRepoDeriveInterval reads the re-derive cadence from the +// environment, falling back to the default for an unset, unparseable or +// non-positive value. +func referenceRepoDeriveInterval() time.Duration { + raw := strings.TrimSpace(os.Getenv(referenceRepoDeriveIntervalEnv)) + if raw == "" { + return referenceRepoDeriveIntervalDefault + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return referenceRepoDeriveIntervalDefault + } + return d +} + +// requestIssueLister lists the open issues in an authorized repo carrying +// label. Production is github.Client.ListRequestIssueCandidates. +type requestIssueLister func(ctx context.Context, repo, label string, limit int) ([]github.RequestIssueCandidate, error) + +// requestIssueAccepterFunc reads who applied label to repo#number and their +// permission. Production is github.Client.RequestIssueAccepterFor. +type requestIssueAccepterFunc func(ctx context.Context, repo string, number int, label string) (github.RequestIssueAccepter, error) + +// intakeResolver returns the intake declaration of the workflow model bound +// to an authorized repo ("owner/name"), and false when the repo is unbound, +// the model is unusable or it declares no intake.referenceRepoFields. +type intakeResolver func(repo string) (workflowmodel.Intake, bool) + +// workflowIntakeResolver resolves bindings the same way the stage engine does +// (stageEngineWorkflowFor): cfg is read on every call so a reload or a +// dashboard binding edit takes effect on the next pass, and the model is read +// from the store rather than compiled because the intake declaration lives on +// the model's spec, not on the compiled profile. +func workflowIntakeResolver(cfg *config.Config, st *wfstore.Store) intakeResolver { + return func(repo string) (workflowmodel.Intake, bool) { + if cfg == nil || st == nil || !cfg.Workflow.Enabled { + return workflowmodel.Intake{}, false + } + b, ok := cfg.WorkflowBindingFor(repo) + if !ok { + return workflowmodel.Intake{}, false + } + m, _, err := st.Get(b.Name, b.Version) + if err != nil || m == nil { + return workflowmodel.Intake{}, false + } + return m.Spec.Intake, m.Spec.Intake.DerivesReferenceRepos() + } +} + +// accepterCacheEntry is the accepter read for one request issue, valid while +// the issue's updated_at has not moved. +type accepterCacheEntry struct { + updatedAt time.Time + accepter github.RequestIssueAccepter +} + +// referenceRepoDeriver runs derivation passes. It remembers the accepter per +// issue so an unchanged issue costs no timeline or permission read, and the +// last good entries per authorized repo so a listing error keeps the previous +// grant instead of dropping it. +type referenceRepoDeriver struct { + list requestIssueLister + accepter requestIssueAccepterFunc + intakeFor intakeResolver + // host is the forge host repository URLs must be on (refderive.Options). + host string + + mu sync.Mutex + // cache is keyed by the issue ref ("owner/repo#N"). + cache map[string]accepterCacheEntry + // lastByRepo holds the entries the previous pass derived from each + // authorized repo's issues, unresolved (as refderive produced them). + lastByRepo map[string][]config.ReferenceRepo +} + +func newReferenceRepoDeriver(list requestIssueLister, accepter requestIssueAccepterFunc, intakeFor intakeResolver, host string) *referenceRepoDeriver { + return &referenceRepoDeriver{ + list: list, + accepter: accepter, + intakeFor: intakeFor, + host: host, + cache: map[string]accepterCacheEntry{}, + lastByRepo: map[string][]config.ReferenceRepo{}, + } +} + +// authorizedRepoSlugs is project.repos as qualified "owner/name" slugs. +func authorizedRepoSlugs(cfg *config.Config) []string { + if cfg == nil { + return nil + } + out := make([]string, 0, len(cfg.Project.Repos)) + for _, repo := range cfg.Project.Repos { + out = append(out, config.QualifyRepo(cfg.Project.Org, repo)) + } + return out +} + +// bodyNamesIntakeField is the cheap pre-filter: an issue whose body carries +// none of the model's reference-repo headings was not filed from the request +// form, so its timeline and the accepter's permission are never fetched. +func bodyNamesIntakeField(body string, intake workflowmodel.Intake) bool { + lower := strings.ToLower(body) + for _, f := range intake.ReferenceRepoFields { + f = strings.ToLower(strings.TrimSpace(f)) + if f != "" && strings.Contains(lower, f) { + return true + } + } + return false +} + +// entriesFrom returns the entries in list derived from the issue ref. +func entriesFrom(list []config.ReferenceRepo, ref string) []config.ReferenceRepo { + var out []config.ReferenceRepo + for _, rr := range list { + if rr.DerivedFrom == ref { + out = append(out, rr) + } + } + return out +} + +// derive runs one pass over every authorized repo whose bound model derives +// reference repos and returns the derived entries, unresolved, in authorized +// repo order then issue order. A nil deriver derives nothing. Errors never +// propagate: a repo whose listing failed keeps its previous entries, an +// issue whose accepter could not be read keeps its previous entries, and a +// refused grant is logged at debug and yields nothing. +func (d *referenceRepoDeriver) derive(ctx context.Context, cfg *config.Config, logger *slog.Logger) []config.ReferenceRepo { + if d == nil || cfg == nil || d.list == nil || d.accepter == nil || d.intakeFor == nil { + return nil + } + if logger == nil { + logger = slog.Default() + } + d.mu.Lock() + defer d.mu.Unlock() + authorized := authorizedRepoSlugs(cfg) + opts := refderive.Options{Host: d.host, Authorized: authorized} + seenIssues := map[string]bool{} + nextByRepo := make(map[string][]config.ReferenceRepo, len(d.lastByRepo)) + var out []config.ReferenceRepo + for _, full := range authorized { + intake, ok := d.intakeFor(full) + if !ok { + continue + } + previous := d.lastByRepo[full] + cands, err := d.list(ctx, full, refderive.GrantLabel, maxRequestIssuesPerRepoPerPass) + if err != nil { + logger.Warn("derived reference repos: request issues not listed, keeping previous entries", "repo", full, "kept", len(previous), "error", err.Error()) + for _, rr := range previous { + seenIssues[rr.DerivedFrom] = true + } + nextByRepo[full] = previous + out = append(out, previous...) + continue + } + var entries []config.ReferenceRepo + for _, cand := range cands { + if cand.Closed || !bodyNamesIntakeField(cand.Body, intake) { + continue + } + issue := refderive.RequestIssue{ + Repo: cand.Repo, + Number: cand.Number, + Title: cand.Title, + Body: cand.Body, + Closed: cand.Closed, + Labels: cand.Labels, + } + ref := issue.Ref() + seenIssues[ref] = true + cached, hit := d.cache[ref] + if !hit || !cached.updatedAt.Equal(cand.UpdatedAt) { + acc, err := d.accepter(ctx, cand.Repo, cand.Number, refderive.GrantLabel) + if err != nil { + kept := entriesFrom(previous, ref) + logger.Warn("derived reference repos: accepter not read, keeping previous entries for the issue", "issue", ref, "kept", len(kept), "error", err.Error()) + entries = append(entries, kept...) + continue + } + cached = accepterCacheEntry{updatedAt: cand.UpdatedAt, accepter: acc} + d.cache[ref] = cached + } + issue.AcceptedBy = cached.accepter.Login + issue.AcceptedByIsBot = cached.accepter.IsBot + issue.AcceptedByPermission = cached.accepter.Permission + res, err := refderive.Derive(issue, intake, opts) + if err != nil { + logger.Debug("derived reference repos: request issue grants nothing", "issue", ref, "reason", err.Error()) + continue + } + for _, sk := range res.Skipped { + logger.Debug("derived reference repos: repository skipped", "issue", ref, "repo", sk.Repo, "reason", sk.Reason) + } + entries = append(entries, res.Repos...) + } + nextByRepo[full] = entries + out = append(out, entries...) + } + for ref := range d.cache { + if !seenIssues[ref] { + delete(d.cache, ref) + } + } + d.lastByRepo = nextByRepo + return out +} + +// clampDerivedWrite returns write restricted to the surfaces a derived entry +// may ever carry (refderive.DerivedWrite): the parser only emits that tier, +// the merge asserts it. +func clampDerivedWrite(write []string) []string { + var out []string + for _, w := range write { + for _, allowed := range refderive.DerivedWrite { + if strings.EqualFold(strings.TrimSpace(w), allowed) { + out = append(out, allowed) + break + } + } + } + return out +} + +// mergeReferenceRepos is the pure merge of derived entries under the +// operator's hand-written list. Hand-written entries come first and +// unchanged; a derived entry is dropped when a hand-written entry names the +// same repo (the operator's tier wins), when the repo is an authorized repo +// (never both), when it carries no provenance, or when an earlier derived +// entry already named the repo (first wins; the other is logged). Every kept +// derived entry's Write is clamped to issues+prs. Other derived fields +// (CredentialState, CredentialReason) pass through so a resolved entry can be +// re-merged after a reload without a new credential lookup. +func mergeReferenceRepos(handWritten, derived []config.ReferenceRepo, authorized []string, logger *slog.Logger) []config.ReferenceRepo { + if logger == nil { + logger = slog.Default() + } + key := func(repo string) string { return strings.ToLower(strings.TrimSpace(repo)) } + taken := make(map[string]string, len(handWritten)+len(derived)) + out := make([]config.ReferenceRepo, 0, len(handWritten)+len(derived)) + for _, rr := range handWritten { + taken[key(rr.Repo)] = "project.reference_repos" + out = append(out, rr) + } + authorizedKeys := make(map[string]bool, len(authorized)) + for _, a := range authorized { + authorizedKeys[key(a)] = true + } + for _, rr := range derived { + k := key(rr.Repo) + switch { + case k == "" || !rr.Derived(): + logger.Warn("derived reference repos: entry without provenance dropped", "repo", rr.Repo) + continue + case authorizedKeys[k]: + logger.Info("derived reference repos: authorized repo is never a reference repo", "repo", rr.Repo, "from", rr.DerivedFrom) + continue + } + if by, dup := taken[k]; dup { + logger.Info("derived reference repos: repo already listed, first wins", "repo", rr.Repo, "from", rr.DerivedFrom, "kept", by) + continue + } + taken[k] = rr.DerivedFrom + rr.Write = clampDerivedWrite(rr.Write) + rr.WriteTokenFile = "" + out = append(out, rr) + } + return out +} + +// referenceRepoRuntime is the state the lifecycle keeps between passes: the +// seams, the deriver and the derived entries as last resolved, so a config +// reload can put them back in memory before the next pass re-lists. +type referenceRepoRuntime struct { + deriver *referenceRepoDeriver + deps referenceRepoDeps + post derivedReadOnlyPoster + // interval is the periodic re-derive cadence; zero disables the poller. + interval time.Duration + + mu sync.Mutex + // lastDerived are the derived entries as buildReferenceRepoPolicies + // resolved them on the last successful pass. + lastDerived []config.ReferenceRepo +} + +// derivedEntries returns the derived entries of a resolved list. +func derivedEntries(refs []config.ReferenceRepo) []config.ReferenceRepo { + var out []config.ReferenceRepo + for _, rr := range refs { + if rr.Derived() { + out = append(out, rr) + } + } + return out +} + +// mergedReferenceRepos validates the merge against the live project; a merge +// that would not load from a file falls back to the hand-written list. +func mergedReferenceRepos(cfg *config.Config, handWritten, derived []config.ReferenceRepo, logger *slog.Logger) []config.ReferenceRepo { + merged := mergeReferenceRepos(handWritten, derived, authorizedRepoSlugs(cfg), logger) + check := &config.Config{Project: config.ProjectConfig{Org: cfg.Project.Org, Repos: cfg.Project.Repos, ReferenceRepos: merged}} + if err := check.ValidateReferenceRepos(); err != nil { + logger.Warn("derived reference repos: merged list invalid, serving hand-written entries only", "error", err.Error()) + return handWritten + } + return merged +} + +// applyReferenceRepos runs one full pass: derive, merge, resolve, install. +// On success the resolved list replaces cfg.Project.ReferenceRepos in memory, +// the proxy gets the rebuilt policies (so an entry that vanished loses its +// policy at once) and the read-only notices go out. A credential error on a +// hand-written write-mode entry is returned for the caller to judge: boot +// exits, a later pass keeps the previous installation. +func (b *boot) applyReferenceRepos(ctx context.Context, rt *referenceRepoRuntime) error { + if rt == nil || b.cfg == nil || b.githubProxy == nil { + return nil + } + rt.mu.Lock() + defer rt.mu.Unlock() + handWritten := config.HandWrittenReferenceRepos(b.cfg.Project.ReferenceRepos) + derived := rt.deriver.derive(ctx, b.cfg, b.logger) + merged := mergedReferenceRepos(b.cfg, handWritten, derived, b.logger) + res, err := buildReferenceRepoPolicies(ctx, merged, rt.deps, b.logger) + if err != nil { + return err + } + // The resolved entries replace the list in memory only: a derived entry + // is never written back to any config file (yaml:"-" on its provenance + // and the save test in reference_repos_derive_test.go guard that), and + // its credential state is what the kick and /api/status read. + b.cfg.Project.ReferenceRepos = res.repos + b.githubProxy.SetReferenceRepos(res.policies) + rt.lastDerived = derivedEntries(res.repos) + b.logger.Info("reference repos wired", "count", len(res.policies), "derived", len(rt.lastDerived), "repos", b.cfg.ReferenceRepoSlugs()) + // Off the caller's path: the notice is a GitHub write and must not delay + // the proxy coming up or a reload completing. + go notifyDerivedReadOnly(ctx, res.repos, rt.post, b.logger) + return nil +} + +// rederiveReferenceRepos is the pass a reload or the poller runs: never +// fatal, the previous installation stays when it fails. +func (b *boot) rederiveReferenceRepos(ctx context.Context) { + rt := b.referenceRepos + if rt == nil { + return + } + if err := b.applyReferenceRepos(ctx, rt); err != nil { + b.logger.Warn("reference repos: re-derive failed, previous policies kept", "error", err.Error()) + } +} + +// reconcileReferenceReposAfterReload runs inside the config reload hook, +// after the new config was swapped in: the reload rebuilt +// cfg.Project.ReferenceRepos from the file, so the derived entries the last +// pass resolved are merged back at once (no network, so the kick never loses +// them while the pass runs), then a full pass re-lists the request issues in +// the background so a grant that ended drops without an operator action. +func (b *boot) reconcileReferenceReposAfterReload() { + if b.referenceRepos == nil || b.cfg == nil { + return + } + b.remergeLastDerivedReferenceRepos() + go b.rederiveReferenceRepos(b.ctx) +} + +// remergeLastDerivedReferenceRepos puts the derived entries of the last +// successful pass back under the hand-written list now in cfg. +func (b *boot) remergeLastDerivedReferenceRepos() { + rt := b.referenceRepos + if rt == nil || b.cfg == nil { + return + } + rt.mu.Lock() + defer rt.mu.Unlock() + handWritten := config.HandWrittenReferenceRepos(b.cfg.Project.ReferenceRepos) + b.cfg.Project.ReferenceRepos = mergedReferenceRepos(b.cfg, handWritten, rt.lastDerived, b.logger) +} + +// startReferenceRepoDerivePoller re-derives on rt.interval until ctx ends. +func (b *boot) startReferenceRepoDerivePoller(ctx context.Context, rt *referenceRepoRuntime) { + if rt == nil || rt.deriver == nil || rt.interval <= 0 || ctx == nil { + return + } + go func() { + ticker := time.NewTicker(rt.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.rederiveReferenceRepos(ctx) + } + } + }() +} diff --git a/src/cmd/hive/reference_repos_derive_test.go b/src/cmd/hive/reference_repos_derive_test.go new file mode 100644 index 0000000000..40582d12e0 --- /dev/null +++ b/src/cmd/hive/reference_repos_derive_test.go @@ -0,0 +1,422 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/refderive" + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// Fixture: the CNCF TechDocs shape. cncf/techdocs is the authorized repo, +// bound to a model whose intake names the form headings; request issue #395 +// names acme/docs (App installed on acme: write tier) and zeta/site (no +// installation: read-only). +const ( + testIntakeRepoField = "Website or documentation repository" + testIntakeOutOfScope = "Out of scope" + testRequestIssueRepo = "cncf/techdocs" + testRequestIssueRef = "cncf/techdocs#395" + testDerivedWriteRepo = "acme/docs" + testDerivedReadOnly = "zeta/site" + testHandWrittenRepo = "hivecommons/docs" + testRequestIssueBody = "### Project\nAcme\n\n### " + testIntakeRepoField + "\nhttps://github.com/acme/docs\nhttps://github.com/zeta/site\nhttps://github.com/cncf/toc\n\n### " + testIntakeOutOfScope + "\n_No response_\n" + testRequestIssueTitle = "Assessment request: Acme" +) + +// fakeRequestIssues is the mutable issue store behind the lister and +// accepter seams; tests close issues, drop labels or move updated_at and +// run another pass. +type fakeRequestIssues struct { + mu sync.Mutex + issues map[int]github.RequestIssueCandidate + accepter github.RequestIssueAccepter + listErr error + accepterCalls int +} + +func newFakeRequestIssues() *fakeRequestIssues { + return &fakeRequestIssues{ + issues: map[int]github.RequestIssueCandidate{ + 395: { + Repo: testRequestIssueRepo, + Number: 395, + Title: testRequestIssueTitle, + Body: testRequestIssueBody, + Labels: []string{refderive.GrantLabel, "kind/assessment"}, + UpdatedAt: time.Date(2026, 11, 1, 10, 0, 0, 0, time.UTC), + }, + }, + accepter: github.RequestIssueAccepter{Login: "maintainer", Permission: "maintain"}, + } +} + +func (f *fakeRequestIssues) list(_ context.Context, repo, label string, limit int) ([]github.RequestIssueCandidate, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.listErr != nil { + return nil, f.listErr + } + var out []github.RequestIssueCandidate + for _, is := range f.issues { + if is.Repo != repo || is.Closed || len(out) >= limit { + continue + } + for _, l := range is.Labels { + if l == label { + out = append(out, is) + break + } + } + } + return out, nil +} + +func (f *fakeRequestIssues) accept(_ context.Context, _ string, _ int, _ string) (github.RequestIssueAccepter, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.accepterCalls++ + return f.accepter, nil +} + +func (f *fakeRequestIssues) update(number int, fn func(*github.RequestIssueCandidate)) { + f.mu.Lock() + defer f.mu.Unlock() + is := f.issues[number] + fn(&is) + f.issues[number] = is +} + +func (f *fakeRequestIssues) accepterCallCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.accepterCalls +} + +// testIntakeFor binds cncf/techdocs to a model with intake fields and +// nothing else. +func testIntakeFor(repo string) (workflowmodel.Intake, bool) { + if repo != testRequestIssueRepo { + return workflowmodel.Intake{}, false + } + return workflowmodel.Intake{ReferenceRepoFields: []string{testIntakeRepoField}, OutOfScopeFields: []string{testIntakeOutOfScope}}, true +} + +func derivedTestConfig(t *testing.T, handWritten ...config.ReferenceRepo) *config.Config { + t.Helper() + return &config.Config{ + SourcePath: filepath.Join(t.TempDir(), "hive.yaml"), + Project: config.ProjectConfig{Org: "cncf", Name: "techdocs", Repos: []string{"techdocs", "toc"}, ReferenceRepos: handWritten}, + AgentRuntimeConfig: config.AgentRuntimeConfig{ + Agents: map[string]config.AgentConfig{"scanner": {Role: "scanner"}}, + }, + } +} + +// derivedTestBoot wires a boot with a real (ephemeral) proxy and the fake +// issue store. deps install the App on acme and hivecommons only. +func derivedTestBoot(t *testing.T, fake *fakeRequestIssues, post derivedReadOnlyPoster, handWritten ...config.ReferenceRepo) (*boot, *referenceRepoRuntime) { + t.Helper() + cfg := derivedTestConfig(t, handWritten...) + px, err := proxy.NewGitHubProxyEphemeral(testLogger(), cfg.Project.Org, cfg.Project.Repos) + if err != nil { + t.Fatal(err) + } + deps := testReferenceRepoDeps() + deps.discover = func(_ context.Context, org string) (int64, error) { + switch org { + case "acme", "hivecommons": + return 1, nil + } + return 0, github.ErrNoInstallationForOrg + } + b := &boot{cfg: cfg, logger: testLogger(), ctx: context.Background(), githubProxy: px} + rt := &referenceRepoRuntime{ + deriver: newReferenceRepoDeriver(fake.list, fake.accept, testIntakeFor, refderive.DefaultHost), + deps: deps, + post: post, + } + return b, rt +} + +func referenceRepoByName(refs []config.ReferenceRepo, repo string) (config.ReferenceRepo, bool) { + for _, rr := range refs { + if strings.EqualFold(rr.Repo, repo) { + return rr, true + } + } + return config.ReferenceRepo{}, false +} + +// Boot derives from the accepted issue: the write-tier entry reaches the +// config and the proxy at issues+prs with its provenance, the uninstalled +// org's entry is read-only and the request issue is told once, and the +// authorized repo the form also named is never derived. +func TestWireReferenceReposDerivesAtBoot(t *testing.T) { + fake := newFakeRequestIssues() + rec := ¬ifyRecorder{posted: true} + b, rt := derivedTestBoot(t, fake, rec.post, config.ReferenceRepo{Repo: testHandWrittenRepo}) + b.wireReferenceReposWith(rt) + + refs := b.cfg.Project.ReferenceRepos + if len(refs) != 3 { + t.Fatalf("want hand-written + 2 derived, got %+v", refs) + } + if refs[0].Repo != testHandWrittenRepo || refs[0].Derived() { + t.Errorf("hand-written entry must come first and unchanged: %+v", refs[0]) + } + docs, ok := referenceRepoByName(refs, testDerivedWriteRepo) + if !ok || docs.DerivedFrom != testRequestIssueRef || !docs.WritesIssues() || !docs.WritesPRs() || docs.CredentialState != config.ReferenceCredentialWriteTier { + t.Errorf("derived write-tier entry = %+v", docs) + } + site, ok := referenceRepoByName(refs, testDerivedReadOnly) + if !ok || !site.DerivedReadOnly() || !site.ReadOnly() { + t.Errorf("derived entry for an uninstalled org must be read-only: %+v", site) + } + if _, derived := referenceRepoByName(refs, "cncf/toc"); derived { + t.Error("an authorized repo named by the form must never become a reference repo") + } + pol, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo) + if !ok || !pol.Issues || !pol.PRs { + t.Errorf("proxy policy for the derived repo = %+v ok=%v", pol, ok) + } + if pol, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedReadOnly); !ok || pol.Issues || pol.PRs { + t.Errorf("proxy policy for the read-only derived repo = %+v ok=%v", pol, ok) + } + testutil.Eventually(t, referenceRepoTestTimeout, func() bool { return rec.count() == 1 }, "read-only notice must be posted once") + rec.mu.Lock() + calls := rec.calls + rec.mu.Unlock() + if len(calls) != 1 || calls[0].repo != testRequestIssueRepo || calls[0].number != 395 || len(calls[0].repos) != 1 || calls[0].repos[0] != testDerivedReadOnly { + t.Errorf("notice calls = %+v", calls) + } +} + +// The grant ends with the label: the next pass drops the entry from the +// config and the proxy, which then refuses writes to the repo again. +func TestRederiveDropsEntryWhenLabelRemoved(t *testing.T) { + fake := newFakeRequestIssues() + b, rt := derivedTestBoot(t, fake, nil) + b.wireReferenceReposWith(rt) + if _, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo); !ok { + t.Fatal("precondition: derived policy installed") + } + + fake.update(395, func(is *github.RequestIssueCandidate) { + is.Labels = []string{"kind/assessment"} + is.UpdatedAt = is.UpdatedAt.Add(time.Minute) + }) + b.rederiveReferenceRepos(context.Background()) + + if len(b.cfg.Project.ReferenceRepos) != 0 { + t.Fatalf("label removed: entries must drop, got %+v", b.cfg.Project.ReferenceRepos) + } + if _, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo); ok { + t.Fatal("label removed: proxy must no longer hold a policy for the repo") + } +} + +func TestRederiveDropsEntryWhenIssueClosed(t *testing.T) { + fake := newFakeRequestIssues() + b, rt := derivedTestBoot(t, fake, nil) + b.wireReferenceReposWith(rt) + + fake.update(395, func(is *github.RequestIssueCandidate) { is.Closed = true }) + b.rederiveReferenceRepos(context.Background()) + + if len(b.cfg.Project.ReferenceRepos) != 0 { + t.Fatalf("closed issue: entries must drop, got %+v", b.cfg.Project.ReferenceRepos) + } + if _, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo); ok { + t.Fatal("closed issue: proxy must no longer hold a policy for the repo") + } +} + +// A hand-written entry for the same repo wins: the operator's read-only +// acme/docs stays read-only, the derived issues+prs grant is dropped. +func TestWireReferenceReposHandWrittenWins(t *testing.T) { + fake := newFakeRequestIssues() + b, rt := derivedTestBoot(t, fake, nil, config.ReferenceRepo{Repo: testDerivedWriteRepo}) + b.wireReferenceReposWith(rt) + + var matches int + for _, rr := range b.cfg.Project.ReferenceRepos { + if strings.EqualFold(rr.Repo, testDerivedWriteRepo) { + matches++ + if rr.Derived() || !rr.ReadOnly() { + t.Errorf("hand-written entry must win unchanged, got %+v", rr) + } + } + } + if matches != 1 { + t.Fatalf("repo listed %d times, want once", matches) + } + if pol, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo); !ok || pol.Issues || pol.PRs { + t.Errorf("proxy must enforce the hand-written read-only tier, got %+v ok=%v", pol, ok) + } +} + +// Refused grants: a bot applier and a triage-only applier derive nothing. +func TestDeriveRefusesBotAndTriageAccepters(t *testing.T) { + for name, acc := range map[string]github.RequestIssueAccepter{ + "bot": {Login: "hive-app[bot]", IsBot: true, Permission: "write"}, + "triage": {Login: "triager", Permission: "triage"}, + "none": {}, + } { + fake := newFakeRequestIssues() + fake.accepter = acc + d := newReferenceRepoDeriver(fake.list, fake.accept, testIntakeFor, refderive.DefaultHost) + if got := d.derive(context.Background(), derivedTestConfig(t), testLogger()); len(got) != 0 { + t.Errorf("%s accepter must grant nothing, got %+v", name, got) + } + } +} + +// The accepter is cached by (issue, updated_at): an unchanged issue costs no +// timeline read on the next pass, a moved one is re-read, and a listing +// error keeps the previous entries. +func TestDeriveCachesAccepterAndKeepsPreviousOnListError(t *testing.T) { + fake := newFakeRequestIssues() + cfg := derivedTestConfig(t) + d := newReferenceRepoDeriver(fake.list, fake.accept, testIntakeFor, refderive.DefaultHost) + ctx := context.Background() + + first := d.derive(ctx, cfg, testLogger()) + if len(first) != 2 || fake.accepterCallCount() != 1 { + t.Fatalf("first pass: %d entries, %d accepter calls", len(first), fake.accepterCallCount()) + } + d.derive(ctx, cfg, testLogger()) + if fake.accepterCallCount() != 1 { + t.Fatalf("unchanged issue must not re-read the accepter, calls = %d", fake.accepterCallCount()) + } + fake.update(395, func(is *github.RequestIssueCandidate) { is.UpdatedAt = is.UpdatedAt.Add(time.Hour) }) + d.derive(ctx, cfg, testLogger()) + if fake.accepterCallCount() != 2 { + t.Fatalf("moved updated_at must re-read the accepter, calls = %d", fake.accepterCallCount()) + } + + fake.mu.Lock() + fake.listErr = errors.New("502 from the forge") + fake.mu.Unlock() + kept := d.derive(ctx, cfg, testLogger()) + if len(kept) != 2 { + t.Fatalf("a listing error must keep the previous entries, got %+v", kept) + } +} + +func TestMergeReferenceRepos(t *testing.T) { + hand := []config.ReferenceRepo{{Repo: "Acme/Docs"}} + derived := []config.ReferenceRepo{ + {Repo: "acme/docs", Write: []string{"issues", "prs"}, DerivedFrom: "o/r#1"}, + {Repo: "zeta/site", Write: []string{"issues", "prs", "admin"}, WriteTokenFile: "/x", DerivedFrom: "o/r#1", CredentialState: config.ReferenceCredentialWriteTier}, + {Repo: "Zeta/Site", Write: []string{"issues"}, DerivedFrom: "o/r#2"}, + {Repo: "cncf/toc", Write: []string{"issues"}, DerivedFrom: "o/r#2"}, + {Repo: "no/provenance", Write: []string{"issues"}}, + } + got := mergeReferenceRepos(hand, derived, []string{"cncf/techdocs", "cncf/toc"}, testLogger()) + if len(got) != 2 { + t.Fatalf("got %+v", got) + } + if got[0].Repo != "Acme/Docs" || got[0].Derived() || !got[0].ReadOnly() { + t.Errorf("hand-written must win unchanged: %+v", got[0]) + } + z := got[1] + if z.Repo != "zeta/site" || z.DerivedFrom != "o/r#1" || z.WriteTokenFile != "" || z.CredentialState != config.ReferenceCredentialWriteTier { + t.Errorf("first derived entry must win with its state: %+v", z) + } + if len(z.Write) != 2 || !z.WritesIssues() || !z.WritesPRs() { + t.Errorf("derived write must be clamped to issues+prs: %v", z.Write) + } + if out := mergeReferenceRepos(nil, nil, nil, nil); len(out) != 0 { + t.Errorf("empty merge = %+v", out) + } +} + +// A derived entry never lands in a saved config: the overlay and runtime +// file writers both go through Config.Save, whose YAML must name the +// hand-written repo and none of the derived ones (hivecommons/hive#18137). +func TestSavedConfigHasNoDerivedReferenceRepos(t *testing.T) { + fake := newFakeRequestIssues() + b, rt := derivedTestBoot(t, fake, nil, config.ReferenceRepo{Repo: testHandWrittenRepo}) + b.wireReferenceReposWith(rt) + if len(b.cfg.Project.ReferenceRepos) != 3 { + t.Fatalf("precondition: derived entries in memory, got %+v", b.cfg.Project.ReferenceRepos) + } + if err := b.cfg.Save(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(b.cfg.SourcePath) + if err != nil { + t.Fatal(err) + } + saved := string(data) + if !strings.Contains(saved, testHandWrittenRepo) { + t.Errorf("saved config must keep the hand-written entry:\n%s", saved) + } + for _, leak := range []string{testDerivedWriteRepo, testDerivedReadOnly, testRequestIssueRef, "derived_from", "credential_state"} { + if strings.Contains(saved, leak) { + t.Errorf("saved config leaks derived state %q:\n%s", leak, saved) + } + } + // The in-memory list is untouched by the save. + if len(b.cfg.Project.ReferenceRepos) != 3 { + t.Errorf("save must not strip the in-memory derived entries: %+v", b.cfg.Project.ReferenceRepos) + } +} + +// A config reload replaces cfg.Project.ReferenceRepos with the file's +// hand-written list; the hook merges the last derived entries back at once +// so the kick never loses them while the background pass runs. +func TestReconcileReferenceReposAfterReloadKeepsDerived(t *testing.T) { + fake := newFakeRequestIssues() + b, rt := derivedTestBoot(t, fake, nil, config.ReferenceRepo{Repo: testHandWrittenRepo}) + b.wireReferenceReposWith(rt) + + // Simulate the reload swap: the file carries only the hand-written entry. + b.cfg.Project.ReferenceRepos = []config.ReferenceRepo{{Repo: testHandWrittenRepo}} + b.remergeLastDerivedReferenceRepos() + refs := b.cfg.Project.ReferenceRepos + if len(refs) != 3 || refs[0].Repo != testHandWrittenRepo { + t.Fatalf("derived entries must be merged back under the hand-written one, got %+v", refs) + } + docs, ok := referenceRepoByName(refs, testDerivedWriteRepo) + if !ok || docs.CredentialState != config.ReferenceCredentialWriteTier || !docs.WritesPRs() { + t.Fatalf("re-merged entry must keep its resolved state, got %+v", docs) + } + + // The full hook also re-lists in the background; with the label gone the + // pass drops the entry from the proxy. + fake.update(395, func(is *github.RequestIssueCandidate) { is.Labels = nil }) + b.reconcileReferenceReposAfterReload() + testutil.Eventually(t, referenceRepoTestTimeout, func() bool { + _, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo) + return !ok + }, "background pass after reload must drop the ended grant from the proxy") +} + +func TestReferenceRepoDeriveInterval(t *testing.T) { + t.Setenv(referenceRepoDeriveIntervalEnv, "") + if got := referenceRepoDeriveInterval(); got != referenceRepoDeriveIntervalDefault { + t.Errorf("unset = %v, want %v", got, referenceRepoDeriveIntervalDefault) + } + t.Setenv(referenceRepoDeriveIntervalEnv, "3m") + if got := referenceRepoDeriveInterval(); got != 3*time.Minute { + t.Errorf("3m = %v", got) + } + for _, bad := range []string{"soon", "-1m", "0"} { + t.Setenv(referenceRepoDeriveIntervalEnv, bad) + if got := referenceRepoDeriveInterval(); got != referenceRepoDeriveIntervalDefault { + t.Errorf("%q = %v, want default", bad, got) + } + } +} diff --git a/src/crux/proxy/reference_repos.go b/src/crux/proxy/reference_repos.go index be69c0c531..11108646d6 100644 --- a/src/crux/proxy/reference_repos.go +++ b/src/crux/proxy/reference_repos.go @@ -140,6 +140,17 @@ func (p *GitHubProxy) SetReferenceRepos(policies []ReferenceRepoPolicy) { p.referenceMu.Unlock() } +// ReferenceRepoPolicyFor returns the installed policy for repo ("owner/name", +// case-insensitive) and whether one exists. Boot tests read it to show that +// a derived entry whose grant ended no longer has a policy +// (hivecommons/hive#18159); the request path uses referenceRepoFor. +func (p *GitHubProxy) ReferenceRepoPolicyFor(repo string) (ReferenceRepoPolicy, bool) { + p.referenceMu.RLock() + defer p.referenceMu.RUnlock() + pol, ok := p.referenceRepos[strings.ToLower(strings.TrimSpace(repo))] + return pol, ok +} + // referenceRepoFor returns the policy covering the repo a path names. func (p *GitHubProxy) referenceRepoFor(path string) (ReferenceRepoPolicy, bool) { p.referenceMu.RLock() diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index d1d4bfe9f8..edce8ef9c7 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -117,6 +117,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_ALLOW_PRIVATE_GIT_SOURCE` | No | `false` | Exact `true` opt-in read by `pkg/knowledge/gitsource.go` to allow knowledge Git sources whose host resolves to a private/internal address (self-hosted GitLab and similar). Off by default as SSRF protection. | | `HIVE_SHARED_AGENT_HOME` | No | per-agent HOME | Escape hatch (`1`) restoring the legacy shared-HOME layout for agents. | | `HIVE_WORKSPACE_CLEANUP_ENABLED` | No | enabled | Set `0` to opt out of automatic agent workspace cleanup. | +| `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` | No | `10m` | How often the hive re-lists accepted request issues to derive `project.reference_repos` entries between config reloads (`src/cmd/hive/reference_repos_derive.go`, #18159). A closed request issue or a removed `triage/accepted` label drops the derived entry on the next pass. Go duration; unset, unparseable or non-positive values fall back to the default. | | `HIVE_WORKSPACE_CLEANUP_INTERVAL` | No | `1h` | How often the workspace cleanup sweep runs (Go duration, e.g. `30m`). Unset, unparseable, or non-positive values fall back to the default. | | `HIVE_WORKSPACE_CLEANUP_MAX_AGE` | No | `2h` | How old an entry under `/data/agents/*/` must be before the cleanup sweep removes it (Go duration, e.g. `6h`). Unset, unparseable, or non-positive values fall back to the default. | | `HIVE_FORGE_COUNT_CACHE_TTL` | No | `20m` | How long the dashboard metrics collector reuses its GitHub Search-API count queries (hive-attributed merged PRs / closed issues per repo for the Cost tiles, outreach PR counts) before re-asking (Go duration, e.g. `30m`). A failed refresh serves the last-good counts (marked stale), retries after 10 minutes and logs at most once per 15 minutes; refreshes are also deferred while the GitHub slow-start caution window is engaged. Unset, unparseable, or non-positive values fall back to the default. | diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index a38e64c09d..d9f8f02fd9 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -8158,6 +8158,12 @@ func (c *Config) redactedForPersist() *Config { if stringSlicesEqual(cp.Governor.Sensing.LoginPatterns, defaultLoginPatterns) { cp.Governor.Sensing.LoginPatterns = nil } + // hivecommons/hive#18159: a reference repo derived from an accepted + // request issue lives in memory only. yaml:"-" hides its provenance, but + // the entry itself would still marshal as a plain repo and outlive the + // grant in the overlay or runtime config (#18137), so the derived entries + // are dropped from the persisted copy here, on every save path. + cp.Project.ReferenceRepos = HandWrittenReferenceRepos(cp.Project.ReferenceRepos) return &cp } diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go index ebee601355..18e2b61d72 100644 --- a/src/pkg/config/reference_repos.go +++ b/src/pkg/config/reference_repos.go @@ -263,6 +263,36 @@ func (c *Config) validateReferenceRepos() error { return nil } +// HandWrittenReferenceRepos returns the entries an operator wrote, i.e. refs +// without the derived ones (hivecommons/hive#18159). nil in, nil out, so a +// config with no reference repos persists without an empty list. +func HandWrittenReferenceRepos(refs []ReferenceRepo) []ReferenceRepo { + if len(refs) == 0 { + return nil + } + out := make([]ReferenceRepo, 0, len(refs)) + for _, r := range refs { + if !r.Derived() { + out = append(out, r) + } + } + if len(out) == 0 { + return nil + } + return out +} + +// ValidateReferenceRepos re-runs the project.reference_repos shape check on +// the live list. The in-memory merge of derived entries +// (hivecommons/hive#18159) calls it after appending, so a derived entry can +// never put the running config in a state a loaded file would be refused in. +func (c *Config) ValidateReferenceRepos() error { + if c == nil { + return nil + } + return c.validateReferenceRepos() +} + // ReferenceRepoSlugs returns the reference repos as sorted "owner/name" // strings, for kicks and environment. func (c *Config) ReferenceRepoSlugs() []string { diff --git a/src/pkg/config/reference_repos_test.go b/src/pkg/config/reference_repos_test.go index 77430ba058..9d91f99217 100644 --- a/src/pkg/config/reference_repos_test.go +++ b/src/pkg/config/reference_repos_test.go @@ -238,3 +238,36 @@ func TestReferenceRepoDerivedFromIssue(t *testing.T) { } } } + +// hivecommons/hive#18159: every save path (runtime config and dashboard +// overlay) marshals redactedForPersist, which must drop derived entries +// outright: yaml:"-" on the provenance alone would still write the repo. +func TestRedactedForPersistDropsDerivedReferenceRepos(t *testing.T) { + cfg := referenceRepoTestConfig(t, ` - hand/written`) + cfg.Project.ReferenceRepos = append(cfg.Project.ReferenceRepos, + ReferenceRepo{Repo: "acme/docs", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395", CredentialState: ReferenceCredentialWriteTier}, + ReferenceRepo{Repo: "zeta/site", DerivedFrom: "cncf/techdocs#395", CredentialState: ReferenceCredentialReadOnly, CredentialReason: "not installed"}, + ) + out, err := yaml.Marshal(cfg.redactedForPersist()) + if err != nil { + t.Fatal(err) + } + s := string(out) + if !strings.Contains(s, "hand/written") { + t.Errorf("hand-written entry must persist:\n%s", s) + } + for _, leak := range []string{"acme/docs", "zeta/site", "techdocs#395", "derived_from", "credential"} { + if strings.Contains(s, leak) { + t.Errorf("persisted config leaks derived state %q:\n%s", leak, s) + } + } + if len(cfg.Project.ReferenceRepos) != 3 { + t.Errorf("redaction must copy, not mutate the live list: %+v", cfg.Project.ReferenceRepos) + } + if got := HandWrittenReferenceRepos(nil); got != nil { + t.Errorf("nil in must be nil out, got %v", got) + } + if got := HandWrittenReferenceRepos([]ReferenceRepo{{Repo: "a/b", DerivedFrom: "o/r#1"}}); got != nil { + t.Errorf("all-derived must be nil, got %v", got) + } +} diff --git a/src/pkg/github/reference_repo_request.go b/src/pkg/github/reference_repo_request.go index dfd0be6f4d..cf310f42b3 100644 --- a/src/pkg/github/reference_repo_request.go +++ b/src/pkg/github/reference_repo_request.go @@ -5,6 +5,9 @@ import ( "fmt" "sort" "strings" + "time" + + gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/pkg/config" ) @@ -115,3 +118,149 @@ func (c *Client) CommentDerivedReadOnly(ctx context.Context, issueRepo string, n } return res.Created, nil } + +// Request-issue discovery for derived reference repos (hivecommons/hive#18159). +// +// The hive derives reference repos from open issues in its authorized repos +// that carry refderive.GrantLabel. Listing is one call per repo; the +// accepter (who applied the label, and with what permission) costs a +// timeline read and a permission lookup per issue, so the caller caches +// those by (issue, updated_at) and only asks again when the issue moved. + +// RequestIssueCandidate is one open issue carrying a grant label, as listed. +// UpdatedAt is GitHub's last-activity timestamp: the caller's cache key. +type RequestIssueCandidate struct { + Repo string + Number int + Title string + Body string + Closed bool + Labels []string + UpdatedAt time.Time +} + +// RequestIssueAccepter is who applied the grant label to a request issue, +// read from the latest "labeled" timeline event for that label. +type RequestIssueAccepter struct { + // Login is the applier; empty when the timeline names no labeled event + // for the label (the label was there before the timeline window, or the + // event carries no actor). + Login string + // IsBot reports a bot or GitHub App account. + IsBot bool + // Permission is Login's permission on the issue's repo as GitHub reports + // it ("admin", "maintain", "write", "triage", "read", "none"); empty when + // Login is empty. + Permission string +} + +// ListRequestIssueCandidates lists the open issues (never pull requests) in +// repo ("owner/name" or bare) that carry label, newest activity first, at +// most limit of them. limit <= 0 means one page. +func (c *Client) ListRequestIssueCandidates(ctx context.Context, repo, label string, limit int) ([]RequestIssueCandidate, error) { + if c == nil || c.client == nil { + return nil, ErrNoGitHubClient + } + owner, name := c.splitRepo(repo) + full := owner + "/" + name + perPage := requestIssueListPageSize + if limit > 0 && limit < perPage { + perPage = limit + } + opts := &gh.IssueListByRepoOptions{ + State: "open", + Labels: []string{label}, + Sort: "updated", + Direction: "desc", + ListOptions: gh.ListOptions{PerPage: perPage}, + } + var out []RequestIssueCandidate + for { + issues, resp, err := c.client.Issues.ListByRepo(ctx, owner, name, opts) + if err != nil { + return nil, fmt.Errorf("listing %s issues labelled %q: %w", full, label, err) + } + for _, is := range issues { + if is == nil || is.IsPullRequest() { + continue + } + cand := RequestIssueCandidate{ + Repo: full, + Number: is.GetNumber(), + Title: is.GetTitle(), + Body: is.GetBody(), + Closed: !strings.EqualFold(is.GetState(), "open"), + Labels: labelNames(is.Labels), + } + if is.UpdatedAt != nil { + cand.UpdatedAt = is.UpdatedAt.Time + } + out = append(out, cand) + if limit > 0 && len(out) >= limit { + return out, nil + } + } + if limit <= 0 || resp == nil || resp.NextPage == 0 { + return out, nil + } + opts.ListOptions.Page = resp.NextPage + } +} + +// requestIssueListPageSize is GitHub's largest issue page. +const requestIssueListPageSize = 100 + +// RequestIssueAccepterFor reads who applied label to repo#number from the +// issue's timeline (the latest "labeled" event for that label wins) and that +// login's permission on repo. An issue whose timeline shows no such event +// yields an empty Login and no error: the caller treats it as an +// unattributed grant. +func (c *Client) RequestIssueAccepterFor(ctx context.Context, repo string, number int, label string) (RequestIssueAccepter, error) { + if c == nil || c.client == nil { + return RequestIssueAccepter{}, ErrNoGitHubClient + } + events, err := c.IssueTimelineSince(ctx, repo, number, time.Time{}) + if err != nil { + return RequestIssueAccepter{}, fmt.Errorf("reading %s#%d timeline: %w", c.qualifiedRepo(repo), number, err) + } + var acc RequestIssueAccepter + for _, ev := range events { + if ev.Event != timelineEventLabeled || !strings.EqualFold(strings.TrimSpace(ev.Label), label) { + continue + } + // Oldest first: the last match is the latest application. + acc = RequestIssueAccepter{Login: strings.TrimSpace(ev.Actor), IsBot: ev.ActorIsBot} + } + if acc.Login == "" { + return acc, nil + } + owner, name := c.splitRepo(repo) + perm, err := c.RepoPermission(ctx, owner, name, acc.Login) + if err != nil { + return RequestIssueAccepter{}, err + } + acc.Permission = perm + return acc, nil +} + +// timelineEventLabeled is GitHub's timeline event name for a label applied. +const timelineEventLabeled = "labeled" + +// RepoPermission returns login's permission level on owner/repo as GitHub +// reports it: "admin", "maintain", "write", "triage", "read" or "none". It +// is the same call reporterHasWriteAccess makes, exported for callers that +// need the level rather than a verdict. +func (c *Client) RepoPermission(ctx context.Context, owner, repo, login string) (string, error) { + if c == nil || c.client == nil { + return "", ErrNoGitHubClient + } + login = strings.TrimSpace(login) + if login == "" { + return "", fmt.Errorf("repo permission on %s/%s: empty login", owner, repo) + } + perm, _, err := c.client.Repositories.GetPermissionLevel(ctx, owner, repo, login) + if err != nil { + return "", fmt.Errorf("reading %s's permission on %s/%s: %w", login, owner, repo, err) + } + return strings.ToLower(strings.TrimSpace(perm.GetPermission())), nil +} diff --git a/src/pkg/github/reference_repo_request_test.go b/src/pkg/github/reference_repo_request_test.go index c7f8a23744..461ab16159 100644 --- a/src/pkg/github/reference_repo_request_test.go +++ b/src/pkg/github/reference_repo_request_test.go @@ -2,6 +2,8 @@ package github import ( "context" + "net/http" + "net/http/httptest" "strings" "testing" @@ -92,3 +94,98 @@ func TestCommentDerivedReadOnlyPostsOnce(t *testing.T) { t.Fatal("nil client must refuse") } } + +// requestIssueServer fakes the three GitHub endpoints request-issue +// discovery reads: the labelled issue list, an issue's timeline and a +// collaborator's permission. +func requestIssueServer(t *testing.T, issuesJSON, timelineJSON, permissionJSON string) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/repos/o/r/issues", func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("labels"); got != "triage/accepted" { + t.Errorf("labels filter = %q, want triage/accepted", got) + } + if got := r.URL.Query().Get("state"); got != "open" { + t.Errorf("state filter = %q, want open", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(issuesJSON)) + }) + mux.HandleFunc("/repos/o/r/issues/395/timeline", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(timelineJSON)) + }) + mux.HandleFunc("/repos/o/r/collaborators/maintainer/permission", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(permissionJSON)) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +const requestIssueListJSON = `[ + {"number":395,"title":"Assessment request","body":"### Website or documentation repository\nhttps://github.com/acme/docs\n","state":"open", + "labels":[{"name":"triage/accepted"},{"name":"kind/assessment"}],"updated_at":"2026-11-01T10:00:00Z"}, + {"number":400,"title":"a PR","body":"","state":"open","labels":[{"name":"triage/accepted"}],"updated_at":"2026-11-02T10:00:00Z", + "pull_request":{"url":"https://api.github.com/repos/o/r/pulls/400"}} +]` + +const requestIssueTimelineJSON = `[ + {"event":"labeled","created_at":"2026-10-01T00:00:00Z","label":{"name":"triage/accepted"},"actor":{"login":"hive-app[bot]","type":"Bot"}}, + {"event":"unlabeled","created_at":"2026-10-02T00:00:00Z","label":{"name":"triage/accepted"},"actor":{"login":"maintainer","type":"User"}}, + {"event":"labeled","created_at":"2026-10-03T00:00:00Z","label":{"name":"kind/assessment"},"actor":{"login":"someone","type":"User"}}, + {"event":"labeled","created_at":"2026-10-04T00:00:00Z","label":{"name":"triage/accepted"},"actor":{"login":"maintainer","type":"User"}} +]` + +func TestListRequestIssueCandidates(t *testing.T) { + srv := requestIssueServer(t, requestIssueListJSON, "[]", "{}") + c := testClient(t, srv.URL) + got, err := c.ListRequestIssueCandidates(context.Background(), "o/r", "triage/accepted", 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("pull requests must be skipped; got %d candidates: %+v", len(got), got) + } + cand := got[0] + if cand.Repo != "o/r" || cand.Number != 395 || cand.Closed || !strings.Contains(cand.Body, "acme/docs") { + t.Errorf("candidate = %+v", cand) + } + if len(cand.Labels) != 2 || cand.Labels[0] != "triage/accepted" { + t.Errorf("labels = %v", cand.Labels) + } + if cand.UpdatedAt.IsZero() { + t.Error("updated_at must be carried: it is the cache key") + } +} + +// The latest "labeled" event for the grant label names the accepter, not an +// earlier application by a bot and not a labeled event for another label. +func TestRequestIssueAccepterForLatestLabeledEvent(t *testing.T) { + srv := requestIssueServer(t, "[]", requestIssueTimelineJSON, `{"permission":"maintain","user":{"login":"maintainer"}}`) + c := testClient(t, srv.URL) + acc, err := c.RequestIssueAccepterFor(context.Background(), "o/r", 395, "triage/accepted") + if err != nil { + t.Fatal(err) + } + if acc.Login != "maintainer" || acc.IsBot || acc.Permission != "maintain" { + t.Fatalf("accepter = %+v, want maintainer/maintain/human", acc) + } +} + +func TestRequestIssueAccepterForNoLabeledEvent(t *testing.T) { + srv := requestIssueServer(t, "[]", `[{"event":"commented","created_at":"2026-10-01T00:00:00Z","actor":{"login":"x"}}]`, "{}") + c := testClient(t, srv.URL) + acc, err := c.RequestIssueAccepterFor(context.Background(), "o/r", 395, "triage/accepted") + if err != nil { + t.Fatal(err) + } + if acc.Login != "" || acc.Permission != "" { + t.Fatalf("an unattributed grant must come back empty, got %+v", acc) + } +} diff --git a/src/pkg/scheduler/agent_repo_scope_test.go b/src/pkg/scheduler/agent_repo_scope_test.go index 4d066fedf2..e243c21fc5 100644 --- a/src/pkg/scheduler/agent_repo_scope_test.go +++ b/src/pkg/scheduler/agent_repo_scope_test.go @@ -370,3 +370,28 @@ func TestBuildReposSectionFor_ItemCitingOnlyReferenceReposIsInScope(t *testing.T } } } + +// hivecommons/hive#18159: a derived entry says which request issue granted +// it, and a derived entry served read-only says why, so an agent can tell a +// grant from an operator's decision. +func TestBuildReposSectionFor_ReferenceReposDerivedProvenance(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "acme/docs", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier}, + {Repo: "acme/site", DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialReadOnly, CredentialReason: "the hive's GitHub App is not installed on acme"}, + } + section := s.buildReposSectionFor("scanner") + for _, want := range []string{ + " https://github.com/hivecommons/docs [reference · read-only]\n", + " https://github.com/acme/docs [reference · issues+prs] (from cncf/techdocs#395)\n", + " https://github.com/acme/site [reference · read-only] (from cncf/techdocs#395; read-only: the hive's GitHub App is not installed on acme)\n", + } { + if !strings.Contains(section, want) { + t.Errorf("REFERENCE REPOS block lacks %q:\n%s", want, section) + } + } + if got := referenceRepoProvenance(config.ReferenceRepo{Repo: "x/y", DerivedFrom: "o/r#1", CredentialState: config.ReferenceCredentialReadOnly}); !strings.Contains(got, derivedReadOnlyNoReason) { + t.Errorf("read-only without a reason must fall back to the generic one, got %q", got) + } +} diff --git a/src/pkg/scheduler/kickmessage.go b/src/pkg/scheduler/kickmessage.go index e1ee2ed765..cd7e5887f9 100644 --- a/src/pkg/scheduler/kickmessage.go +++ b/src/pkg/scheduler/kickmessage.go @@ -854,7 +854,7 @@ func (s *Scheduler) buildReposSectionFor(agentName string) string { if refs := s.cfg.Project.ReferenceRepos; len(refs) > 0 { b.WriteString("📚 REFERENCE REPOS (outside this project — study them, do not work them):\n") for _, rr := range refs { - b.WriteString(fmt.Sprintf(" %s/%s [reference · %s]\n", strings.TrimRight(host, "/"), rr.Repo, rr.WriteLabel())) + b.WriteString(fmt.Sprintf(" %s/%s [reference · %s]%s\n", strings.TrimRight(host, "/"), rr.Repo, rr.WriteLabel(), referenceRepoProvenance(rr))) } b.WriteString(" You MAY clone them (public, unauthenticated), read their files, and GET their issues/PRs/API. An authorized-repo item whose deliverables name a reference repo is IN SCOPE, not a scope violation: do NOT classify it out-of-scope, critical, or a security concern. Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo (the assessment document, the report), never as issues or PRs on the reference repo itself — unless its tier above says issues/prs, in which case you may ONLY open an issue, open a PR from your own branch, or comment on an issue/PR you opened; the proxy refuses merges, default-branch pushes, labels, assignments, reviews and closing others' items deterministically, so retrying cannot succeed.\n") } @@ -910,3 +910,25 @@ Your workdir is, at most, a checkout of the primary repo — never of the others } return b.String() } + +// derivedReadOnlyNoReason is the kick's fallback when a read-only derived +// reference repo carries no CredentialReason. +const derivedReadOnlyNoReason = "no write credential for this repository" + +// referenceRepoProvenance is the suffix the REFERENCE REPOS block appends +// for an entry derived from an accepted request issue +// (hivecommons/hive#18159): where the grant came from, and for one served +// read-only, why. Empty for a hand-written entry. +func referenceRepoProvenance(rr config.ReferenceRepo) string { + if !rr.Derived() { + return "" + } + if !rr.DerivedReadOnly() { + return " (from " + rr.DerivedFrom + ")" + } + reason := strings.TrimSpace(rr.CredentialReason) + if reason == "" { + reason = derivedReadOnlyNoReason + } + return " (from " + rr.DerivedFrom + "; read-only: " + reason + ")" +} From 4f1dfd2268c83729d1a0730df0b9b479b0287996 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 21:59:46 -0500 Subject: [PATCH 205/223] =?UTF-8?q?=E2=9C=A8=20spoke:=20report-only=20hear?= =?UTF-8?q?tbeat=20mirror=20to=20a=20second=20hub=20(HIVE=5FMIRROR=5FHUB?= =?UTF-8?q?=5FURL)=20(#18173)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the primary hub accepts a beat, a spoke with HIVE_MIRROR_HUB_URL set POSTs a reduced copy of the same payload to the mirror hub with a separate per-hive key (HIVE_MIRROR_HUB_HEARTBEAT_KEY), in its own goroutine with its own short timeout. The copy drops the dashboard token hash, App and webhook key fingerprints, authentication error text, signed-in users and the activity digest (heartbeatMirrorFieldPolicy, inventory-tested). The mirror response is never applied; authority fields in it are counted, surfaced under hub_heartbeat.mirror in /api/health/deep, and logged once per hour as mirror_hub_response_ignored. A mirror failure never affects the primary heartbeat or the hive. Fixes #18148 Part of #18060 Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-18148-mirror-hub.md | 1 + src/docs/env-vars.md | 4 + src/pkg/dashboard/server.go | 7 + src/pkg/hub/spoke/heartbeat.go | 5 + src/pkg/hub/spoke/heartbeat_mirror.go | 568 ++++++++++++++++++++ src/pkg/hub/spoke/heartbeat_mirror_test.go | 581 +++++++++++++++++++++ src/pkg/hub/spoke/heartbeat_testhooks.go | 2 + 7 files changed, 1168 insertions(+) create mode 100644 changelog.d/added-18148-mirror-hub.md create mode 100644 src/pkg/hub/spoke/heartbeat_mirror.go create mode 100644 src/pkg/hub/spoke/heartbeat_mirror_test.go diff --git a/changelog.d/added-18148-mirror-hub.md b/changelog.d/added-18148-mirror-hub.md new file mode 100644 index 0000000000..4adf5488b1 --- /dev/null +++ b/changelog.d/added-18148-mirror-hub.md @@ -0,0 +1 @@ +- Spoke: optional **report-only heartbeat mirror** to a second hub ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18148](https://github.com/hivecommons/hive/issues/18148)). With `HIVE_MIRROR_HUB_URL` and `HIVE_MIRROR_HUB_HEARTBEAT_KEY` set (the key comes from registering the hive on that hub, for example through its **Register your own hive** form), every heartbeat the primary hub accepts is also POSTed, in its own goroutine with its own short timeout (`HIVE_MIRROR_HUB_TIMEOUT`, default 5s), to the mirror hub as a reduced copy: the dashboard token hash, the App and webhook key fingerprints, authentication error text, the signed-in user list and the activity digest are left out, and `HIVE_MIRROR_HUB_HIVE_ID` can override the reported `hive_id`. The mirror's response is never applied: upgrade, restart, branch switch, breaker, GitHub App config, funded gateway, project claim, access list, visibility and cluster id from the mirror are counted and ignored, and logged at most once per hour as `mirror_hub_response_ignored` (field names only). A mirror failure never affects the primary heartbeat, liveness or the hive. The primary key is never sent to the mirror; without a mirror key nothing is sent. `GET /api/health/deep` shows the mirror under `hub_heartbeat.mirror` (`url_host`, `last_ok`, `last_attempt`, `last_error`, `ignored_fields`, `last_ignored_fields`), host only, never the key. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index edce8ef9c7..f46c859cd0 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -70,6 +70,10 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_ID` | No | config or generated id | Stable hive/spoke identifier override; passed through to launched agents. | | `HIVE_CLUSTER_ID` | No | config or hub-provisioned value | Hosted cluster identifier override. | | `HIVE_HUB_URL` | No | `hub.url` from config | Hub URL override for spoke heartbeats/registration. On the hub it is also the last environment variable consulted in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | +| `HIVE_MIRROR_HUB_URL` | No | unset (no mirror) | Base URL of a **second hub** that receives a report-only copy of every heartbeat the primary hub accepts ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18148](https://github.com/hivecommons/hive/issues/18148); `pkg/hub/spoke/heartbeat_mirror.go`). Must be an absolute `http(s)` URL and must differ from the primary hub. The copy leaves out the dashboard token hash, the App and webhook key fingerprints, authentication error text, the signed-in user list and the activity digest. The mirror's response is never applied: config, credential, upgrade, restart, breaker, access-list and visibility fields from it are counted and ignored, and logged at most once per hour as `mirror_hub_response_ignored` (field names only). A mirror failure never affects the primary heartbeat, liveness or the hive. Status appears in `GET /api/health/deep` under `hub_heartbeat.mirror` (host only). Read per beat; no restart needed. | +| `HIVE_MIRROR_HUB_HEARTBEAT_KEY` | Required when `HIVE_MIRROR_HUB_URL` is set | none | Per-hive heartbeat key the **mirror** hub issued (its **Register your own hive** form, or an operator-minted key). Sent only to the mirror. The primary `HIVE_HEARTBEAT_KEY` is never offered to a mirror: with the URL set and this unset, nothing is sent and `hub_heartbeat.mirror.last_error` says so. Never logged. | +| `HIVE_MIRROR_HUB_HIVE_ID` | No | the primary `hive_id` | Overrides `hive_id` on the mirrored copy only, for a mirror hub that knows this hive under another id (for example its self-service `byo-<login>-<name>` id). | +| `HIVE_MIRROR_HUB_TIMEOUT` | No | `5s` | Timeout for one mirror POST, Go duration syntax. Deliberately shorter than the primary heartbeat budget; a slow mirror misses a beat rather than holding a goroutine. Invalid or non-positive values keep the default. | | `HIVE_SHARED_AGENT_POOL_ENABLED` | No | `false` | Spoke opt-in for Hub-coordinated CLI session sharing. Requires an in-process runtime and a Hub link; each Hive keeps its own workspace, UID, credentials, and tmux session. Idle CLI sessions are terminated after turns to return the global slot. | | `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` | No | `0` (disabled) | Hub-side maximum number of live CLI session leases. Set to `1` to serialize agent sessions across participating Hives. | | `HIVE_SHARED_AGENT_POOL_MAX_PER_HIVE` | No | `0` (no per-tenant cap) | Hub-side maximum number of live CLI session leases a single Hive may hold. Set below `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` on multi-tenant hosts so one tenant cannot take every slot and renew them indefinitely. A Hive at its cap gets `409` without taking a FIFO position. | diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index ddd90a1e0f..6e7bcd46da 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -3704,6 +3704,13 @@ func (s *Server) handleHealthDeep(w http.ResponseWriter, r *http.Request) { hbCheck["last_attempt"] = lastAttempt.UTC().Format(time.RFC3339) hbCheck["last_attempt_age"] = time.Since(lastAttempt).Round(time.Second).String() } + // Report-only mirror to a second hub (#18148): visibility only. A + // failing mirror never degrades this check, because the mirror has no + // role in liveness or in managing the hive; the owner reads it here to + // confirm the mirror is receiving beats (host only, never the key). + if mirror, ok := spoke.MirrorHubStatusSnapshot(); ok { + hbCheck["mirror"] = mirror + } if hbCheck["status"] == "warn" && overall == "ok" { overall = "degraded" } diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 2ca00c37cd..575ae8b66d 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1897,6 +1897,11 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa // record it even if the response body below fails to decode. recordHeartbeatSuccess() + // Report-only mirror to a second hub (#18148): a reduced copy of this + // accepted beat goes out in its own goroutine with its own timeout. Its + // response is never applied, and nothing below depends on it. + mirrorHeartbeatAsync(ctx, hubURL, payload, logger) + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxPayloadBytes)) if err != nil { logger.Warn("hub heartbeat response read failed", "error", err) diff --git a/src/pkg/hub/spoke/heartbeat_mirror.go b/src/pkg/hub/spoke/heartbeat_mirror.go new file mode 100644 index 0000000000..aa8f03b5b6 --- /dev/null +++ b/src/pkg/hub/spoke/heartbeat_mirror.go @@ -0,0 +1,568 @@ +package spoke + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Report-only heartbeat mirror to a second hub (#18148, RFC #18060 section 6). +// +// A spoke that its own hub manages can also SHOW on a second hub, for example +// a self-hosted hive listed on the public hub through the self-service +// registration flow (#18142, #18155). The mirror is strictly one-way: +// +// - The spoke POSTs a REDUCED copy of each heartbeat to the mirror hub, +// authenticated with a separate per-hive key (EnvMirrorHubHeartbeatKey). +// The primary key is never sent to the mirror, and the mirror key is never +// sent to the primary. +// - The mirror's response is never applied. Nothing in it reaches +// processHeartbeatResponse or any callback: no upgrade, restart, branch +// switch, breaker, GitHub App config, funded gateway, project claim, +// access list, visibility, cluster id or experiment flag. The response is +// read only to COUNT those fields so an owner can see a mirror that is +// trying to manage them (mirrorResponseIgnoredEvent). +// - A mirror failure never affects the primary heartbeat or the hive. The +// POST runs in its own goroutine with its own short timeout, after the +// primary hub has accepted the beat, and never touches the liveness +// clocks (lastHeartbeatSuccessUnix / lastHeartbeatAttemptUnix) or the +// activity publisher's acknowledgement state. +// +// The reduced copy is driven by heartbeatMirrorFieldPolicy, which classifies +// every HeartbeatPayload field as mirrored or dropped; the inventory test +// fails when a field is added without a classification, so a new sensitive +// field cannot reach the mirror by default (same pattern as the hub's +// heartbeatResponseFieldFence). + +const ( + // EnvMirrorHubURL is the base URL of the second hub. Unset disables the + // mirror entirely and nothing in this file runs. + EnvMirrorHubURL = "HIVE_MIRROR_HUB_URL" + // EnvMirrorHubHeartbeatKey is the per-hive heartbeat key the MIRROR hub + // issued (self-service registration, or an operator-minted key). It is + // required: without it no mirror beat is sent, because the primary key must + // never be offered to another hub. + EnvMirrorHubHeartbeatKey = "HIVE_MIRROR_HUB_HEARTBEAT_KEY" + // EnvMirrorHubHiveID optionally overrides hive_id on the mirrored copy, for + // a mirror hub that knows this hive under a different (for example byo-) + // id. Empty keeps the primary hive id. + EnvMirrorHubHiveID = "HIVE_MIRROR_HUB_HIVE_ID" + // EnvMirrorHubTimeout overrides the per-beat mirror POST timeout (Go + // duration syntax). Invalid or non-positive values keep the default. + EnvMirrorHubTimeout = "HIVE_MIRROR_HUB_TIMEOUT" + + // defaultMirrorHubTimeout bounds one mirror POST. Deliberately shorter than + // heartbeatTimeout: the mirror has no liveness role, so a slow mirror hub + // should simply miss a beat rather than hold a goroutine for long. + defaultMirrorHubTimeout = 5 * time.Second + + // mirrorResponseIgnoredEvent is the log event name written at most once per + // mirrorIgnoredLogWindow when the mirror's response carried authority + // fields (upgrade, App config, gateway, ...) that this spoke ignored. + mirrorResponseIgnoredEvent = "mirror_hub_response_ignored" + + // mirrorResponseMaxBytes caps how much of the mirror's response is read. + // Only the authority-field count comes out of it, so a modest cap is plenty + // and a hostile mirror cannot grow spoke memory. + mirrorResponseMaxBytes = 256 * 1024 + + // mirrorErrorBodyMaxBytes caps the rejection body kept for the status block. + mirrorErrorBodyMaxBytes = 256 + + // mirrorHeartbeatPath is the hub route the copy is POSTed to, the same route + // the primary beat uses. + mirrorHeartbeatPath = "/api/heartbeat" +) + +// mirrorIgnoredLogWindow is how often (at most) the ignored-authority log line +// repeats while a mirror keeps sending instructions. A var so tests can shorten +// it; production keeps the default. +var mirrorIgnoredLogWindow = time.Hour + +// mirrorFieldPolicy is how the reducer treats one HeartbeatPayload field. +type mirrorFieldPolicy int + +const ( + // mirrorFieldMirrored is sent to the mirror hub as-is. + mirrorFieldMirrored mirrorFieldPolicy = iota + 1 + // mirrorFieldDropped is zeroed before the copy leaves the spoke: a + // credential-adjacent identifier (token hash, key fingerprints), the text of + // an authentication error, the list of signed-in users, the activity digest, + // or consent that belongs to the primary hub only. + mirrorFieldDropped +) + +// heartbeatMirrorFieldPolicy classifies every HeartbeatPayload field by its Go +// field name. Adding a field to HeartbeatPayload requires adding it here; +// TestHeartbeatMirrorFieldPolicyInventory fails otherwise. Any field missing +// from this table is DROPPED by the reducer, so an unclassified field fails +// closed even in a build that skipped the inventory test. +var heartbeatMirrorFieldPolicy = map[string]mirrorFieldPolicy{ + // Identity and project. + "HiveID": mirrorFieldMirrored, + "Org": mirrorFieldMirrored, + "Repos": mirrorFieldMirrored, + "PrimaryRepo": mirrorFieldMirrored, + "AIAuthor": mirrorFieldMirrored, + "AIAuthorEffective": mirrorFieldMirrored, + "GitHubAPIURL": mirrorFieldMirrored, + "GitHubHost": mirrorFieldMirrored, + "GitHubBaseURL": mirrorFieldMirrored, + "Owner": mirrorFieldMirrored, + "HiveType": mirrorFieldMirrored, + "ClusterID": mirrorFieldMirrored, + "IsPublic": mirrorFieldMirrored, + "Reporter": mirrorFieldMirrored, + "StartedAt": mirrorFieldMirrored, + "Timestamp": mirrorFieldMirrored, + // Build and deployment. + "Version": mirrorFieldMirrored, + "GitHash": mirrorFieldMirrored, + "GitBranch": mirrorFieldMirrored, + "ImageRef": mirrorFieldMirrored, + // Agents, governor, stats. + "ACMMLevel": mirrorFieldMirrored, + "Agents": mirrorFieldMirrored, + "Governor": mirrorFieldMirrored, + "Tokens24h": mirrorFieldMirrored, + "Contributors": mirrorFieldMirrored, + "Leaderboard": mirrorFieldMirrored, + "OpenFDs": mirrorFieldMirrored, + "FDSoftLimit": mirrorFieldMirrored, + "Health": mirrorFieldMirrored, + "DashboardURL": mirrorFieldMirrored, + "SnapshotURL": mirrorFieldMirrored, + "Providers": mirrorFieldMirrored, + "ProviderLimitReason": mirrorFieldMirrored, + "ProviderLimitRebuffs": mirrorFieldMirrored, + "ProviderLimitHiveWide": mirrorFieldMirrored, + "ProviderLimitAgents": mirrorFieldMirrored, + "LastWriteCapableKickAt": mirrorFieldMirrored, + "LastKickDisposition": mirrorFieldMirrored, + "LastKickSkipReason": mirrorFieldMirrored, + "NotWritableQueued": mirrorFieldMirrored, + "PublicURLSelfCheck": mirrorFieldMirrored, + "RouteExists": mirrorFieldMirrored, + "ForgeUnreachable": mirrorFieldMirrored, + "ClusterHealth": mirrorFieldMirrored, + "StatsStale": mirrorFieldMirrored, + "FreshAgentStats": mirrorFieldMirrored, + // Advisory digest telemetry (counts and timestamps; the error text is dropped below). + "AdvisoryLastPostedAt": mirrorFieldMirrored, + "AdvisoryFindingCount": mirrorFieldMirrored, + "AdvisoryOverflowCount": mirrorFieldMirrored, + // GitHub App posture (non-secret state tokens, ids and status words). + "GitHubAppRequired": mirrorFieldMirrored, + "GitHubAppTokenStatus": mirrorFieldMirrored, + "GitHubAppTokenLastMintAt": mirrorFieldMirrored, + "GitHubAppErrorClass": mirrorFieldMirrored, + "GitHubAppHTTPStatus": mirrorFieldMirrored, + "GitHubAppState": mirrorFieldMirrored, + "GitHubAppID": mirrorFieldMirrored, + "GitHubAppSlug": mirrorFieldMirrored, + "GitHubInstallationID": mirrorFieldMirrored, + "GitHubAppKeyPerHive": mirrorFieldMirrored, + "PendingGitHubAppInstall": mirrorFieldMirrored, + "RepoTargetMisconfigured": mirrorFieldMirrored, + "RepoTargetIssue": mirrorFieldMirrored, + // Upgrade state (what the spoke reports about itself). + "AutoUpgrade": mirrorFieldMirrored, + "Upgrading": mirrorFieldMirrored, + "UpgradeTargetSHA": mirrorFieldMirrored, + "UpgradeFailed": mirrorFieldMirrored, + "UpgradeError": mirrorFieldMirrored, + "UpgradePendingRestart": mirrorFieldMirrored, + "UpgradeDrain": mirrorFieldMirrored, + // Fleet contribution, repo activity and quadrant signals. + "PRsMerged90d": mirrorFieldMirrored, + "PRsRejected90d": mirrorFieldMirrored, + "CVEsClosed": mirrorFieldMirrored, + "FleetStatsCollectedAt": mirrorFieldMirrored, + "RepoActivity": mirrorFieldMirrored, + "RepoActivityCollectedAt": mirrorFieldMirrored, + "RepoActivityWindowHours": mirrorFieldMirrored, + "RepoActivityCountWindowHours": mirrorFieldMirrored, + "BudgetCurrentSpend": mirrorFieldMirrored, + "BudgetLimit": mirrorFieldMirrored, + "BudgetWindowStartsAt": mirrorFieldMirrored, + "BudgetWindowEndsAt": mirrorFieldMirrored, + "BudgetExhausted": mirrorFieldMirrored, + "BudgetIgnored": mirrorFieldMirrored, + "HoldTotal": mirrorFieldMirrored, + "AwaitingReview": mirrorFieldMirrored, + "AgentErrorStreaks": mirrorFieldMirrored, + "ConsentWedged": mirrorFieldMirrored, + "NoCadenceAgents": mirrorFieldMirrored, + "SLAViolations": mirrorFieldMirrored, + "TasksCompleted7d": mirrorFieldMirrored, + "AgentsWithModel": mirrorFieldMirrored, + "GatewayNames": mirrorFieldMirrored, + "GatewayHealth": mirrorFieldMirrored, + "ComponentReach": mirrorFieldMirrored, + "Compliance": mirrorFieldMirrored, + "Evidence": mirrorFieldMirrored, + + // Dropped: credential-adjacent identifiers. The primary hub matches these + // against records it holds; a report-only hub has no such records and must + // not learn which key or token this hive carries. + "DashboardTokenHash": mirrorFieldDropped, + "GitHubAppKeyFingerprint": mirrorFieldDropped, + "WebhookRelayKeyFingerprint": mirrorFieldDropped, + "GitHubAppKeysHeld": mirrorFieldDropped, + // Dropped: the text of authentication errors. + "InferenceAuthError": mirrorFieldDropped, + "GitHubAppTokenError": mirrorFieldDropped, + "GitHubAppPermIssue": mirrorFieldDropped, + "AdvisoryError": mirrorFieldDropped, + // Dropped: who is signed in to this hive. + "ActiveSessionUsers": mirrorFieldDropped, + "EngagedSessionUsers": mirrorFieldDropped, + "UserLastActions": mirrorFieldDropped, + // Dropped: the activity digest (its acknowledgement protocol belongs to the + // primary hub's activity publisher). + "Activity": mirrorFieldDropped, + // Dropped: consent to receive flags, which only the primary may deliver. + "SelfExperimentsOptIn": mirrorFieldDropped, +} + +// heartbeatMirrorResponseHarmless names the HeartbeatResponse fields that +// carry no authority over the spoke: protocol, version display and signature +// binding. Every OTHER field of a mirror response is an attempt to deliver +// credentials or instructions and is counted as ignored. Fail-closed by +// construction: a new response field is counted until someone lists it here. +var heartbeatMirrorResponseHarmless = map[string]bool{ + "OK": true, + "HubGitHash": true, + "LatestSHA": true, + "LatestTag": true, + "SigHiveID": true, + "SigSeq": true, + "SigSignedAt": true, + "SigVersion": true, + "ActivityETag": true, + "ActivityEventsAck": true, +} + +// mirrorHubConfig is the mirror's effective configuration, read from the +// environment on every beat (like SpokeHeartbeatKey) so a rotated key or a +// changed URL takes effect without a restart. +type mirrorHubConfig struct { + url string + key string + hiveID string + timeout time.Duration +} + +// mirrorHubConfigured reports whether EnvMirrorHubURL is set at all. It is the +// only check the hot path pays when the mirror is off. +func mirrorHubConfigured() bool { + return strings.TrimSpace(os.Getenv(EnvMirrorHubURL)) != "" +} + +// loadMirrorHubConfig validates the mirror environment. It returns a +// user-facing error (never containing the key) when the mirror is configured +// but unusable, so the status block can say why nothing is being sent. +func loadMirrorHubConfig(primaryHubURL string) (mirrorHubConfig, error) { + raw := strings.TrimRight(strings.TrimSpace(os.Getenv(EnvMirrorHubURL)), "/") + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return mirrorHubConfig{}, fmt.Errorf("%s must be an absolute http(s) URL", EnvMirrorHubURL) + } + if primaryHubURL != "" && strings.TrimRight(strings.TrimSpace(primaryHubURL), "/") == raw { + return mirrorHubConfig{}, fmt.Errorf("%s is the primary hub URL; the mirror must be a different hub", EnvMirrorHubURL) + } + key := strings.TrimSpace(os.Getenv(EnvMirrorHubHeartbeatKey)) + if key == "" { + return mirrorHubConfig{}, fmt.Errorf("%s is not set; the primary heartbeat key is never sent to a mirror", EnvMirrorHubHeartbeatKey) + } + timeout := defaultMirrorHubTimeout + if v := strings.TrimSpace(os.Getenv(EnvMirrorHubTimeout)); v != "" { + if d, perr := time.ParseDuration(v); perr == nil && d > 0 { + timeout = d + } + } + return mirrorHubConfig{ + url: raw, + key: key, + hiveID: strings.TrimSpace(os.Getenv(EnvMirrorHubHiveID)), + timeout: timeout, + }, nil +} + +// MirrorHubStatus is the owner-facing view of the mirror, surfaced in the +// spoke's deep health check under hub_heartbeat.mirror. It names the mirror +// HOST only; the key never appears anywhere observable. +type MirrorHubStatus struct { + // URLHost is the host (and port) of EnvMirrorHubURL, or empty when the + // configured value could not be parsed. + URLHost string `json:"url_host"` + // LastOK is when the mirror last accepted a beat (RFC3339 UTC), empty if never. + LastOK string `json:"last_ok,omitempty"` + // LastAttempt is when a mirror beat was last attempted (RFC3339 UTC). + LastAttempt string `json:"last_attempt,omitempty"` + // LastError is the most recent failure (configuration, transport, or a + // rejection with its HTTP status), cleared on the next accepted beat. + LastError string `json:"last_error,omitempty"` + // IgnoredFields is the running total of authority-carrying fields the + // mirror's responses tried to deliver and this spoke ignored. + IgnoredFields int64 `json:"ignored_fields"` + // LastIgnoredFields lists the JSON names of the authority fields present in + // the most recent mirror response that carried any (names only, never values). + LastIgnoredFields []string `json:"last_ignored_fields,omitempty"` +} + +// mirrorState is the process-global mirror bookkeeping. Written by the mirror +// goroutine, read by the dashboard's health handler. +var mirrorState = struct { + sync.Mutex + lastOK time.Time + lastAttempt time.Time + lastError string + lastIgnoredFields []string + ignoredLoggedAt time.Time +}{} + +// mirrorIgnoredTotal is the running count of ignored authority fields. +var mirrorIgnoredTotal atomic.Int64 + +// mirrorInFlight refuses a second concurrent mirror POST: a stalled mirror hub +// must not accumulate goroutines beat after beat. +var mirrorInFlight atomic.Bool + +// MirrorHubStatusSnapshot returns the mirror's status block and whether a +// mirror is configured at all. ok is false (and the block empty) when EnvMirrorHubURL is +// unset, so callers can omit the block entirely. +func MirrorHubStatusSnapshot() (MirrorHubStatus, bool) { + if !mirrorHubConfigured() { + return MirrorHubStatus{}, false + } + out := MirrorHubStatus{IgnoredFields: mirrorIgnoredTotal.Load()} + if u, err := url.Parse(strings.TrimSpace(os.Getenv(EnvMirrorHubURL))); err == nil { + out.URLHost = u.Host + } + mirrorState.Lock() + defer mirrorState.Unlock() + if !mirrorState.lastOK.IsZero() { + out.LastOK = mirrorState.lastOK.UTC().Format(time.RFC3339) + } + if !mirrorState.lastAttempt.IsZero() { + out.LastAttempt = mirrorState.lastAttempt.UTC().Format(time.RFC3339) + } + out.LastError = mirrorState.lastError + out.LastIgnoredFields = append([]string(nil), mirrorState.lastIgnoredFields...) + return out, true +} + +func recordMirrorAttempt() { + mirrorState.Lock() + mirrorState.lastAttempt = time.Now() + mirrorState.Unlock() +} + +func recordMirrorError(msg string) { + mirrorState.Lock() + mirrorState.lastError = msg + mirrorState.Unlock() +} + +func recordMirrorSuccess() { + mirrorState.Lock() + mirrorState.lastOK = time.Now() + mirrorState.lastError = "" + mirrorState.Unlock() +} + +// resetMirrorStateForTest returns the mirror bookkeeping to process-start values. +func resetMirrorStateForTest() { + mirrorState.Lock() + mirrorState.lastOK = time.Time{} + mirrorState.lastAttempt = time.Time{} + mirrorState.lastError = "" + mirrorState.lastIgnoredFields = nil + mirrorState.ignoredLoggedAt = time.Time{} + mirrorState.Unlock() + mirrorIgnoredTotal.Store(0) + mirrorInFlight.Store(false) +} + +// reduceHeartbeatForMirror returns the copy of p that may leave for the mirror +// hub: a deep clone with every dropped (or unclassified) field zeroed and, when +// hiveIDOverride is non-empty, hive_id replaced. The caller's payload is never +// modified. Returns nil when the payload cannot be cloned. +func reduceHeartbeatForMirror(p *HeartbeatPayload, hiveIDOverride string) *HeartbeatPayload { + c := clonePayload(p) + if c == nil { + return nil + } + v := reflect.ValueOf(c).Elem() + t := v.Type() + for i := 0; i < t.NumField(); i++ { + if heartbeatMirrorFieldPolicy[t.Field(i).Name] == mirrorFieldMirrored { + continue + } + if fv := v.Field(i); fv.CanSet() { + fv.Set(reflect.Zero(fv.Type())) + } + } + if hiveIDOverride != "" { + c.HiveID = hiveIDOverride + } + return c +} + +// mirrorResponseAuthorityFields decodes a mirror hub response and returns the +// JSON names of every non-harmless field it set. It never returns the decoded +// response: nothing in it may be applied. +func mirrorResponseAuthorityFields(raw []byte) ([]string, error) { + var resp HeartbeatResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + v := reflect.ValueOf(resp) + t := v.Type() + var names []string + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if heartbeatMirrorResponseHarmless[f.Name] || v.Field(i).IsZero() { + continue + } + name := strings.Split(f.Tag.Get("json"), ",")[0] + if name == "" { + name = f.Name + } + names = append(names, name) + } + return names, nil +} + +// mirrorHeartbeatAsync sends the reduced copy of payload to the mirror hub in +// its own goroutine. Called from postHeartbeatToHub after the primary hub +// accepted the beat; it returns immediately. payload is read synchronously +// (cloned) before the goroutine starts, so the caller may reuse it. +func mirrorHeartbeatAsync(ctx context.Context, primaryHubURL string, payload *HeartbeatPayload, logger *slog.Logger) { + if payload == nil || !mirrorHubConfigured() { + return + } + cfg, err := loadMirrorHubConfig(primaryHubURL) + if err != nil { + recordMirrorError(err.Error()) + return + } + copyForMirror := reduceHeartbeatForMirror(payload, cfg.hiveID) + if copyForMirror == nil { + recordMirrorError("mirror payload could not be cloned") + return + } + if !mirrorInFlight.CompareAndSwap(false, true) { + recordMirrorError("previous mirror beat still in flight; skipped this beat") + return + } + go func() { + defer mirrorInFlight.Store(false) + postHeartbeatToMirror(ctx, cfg, copyForMirror, logger) + }() +} + +// postHeartbeatToMirror performs one mirror POST. It records the outcome in +// mirrorState and returns nothing: the response is parsed only to count the +// authority fields the mirror tried to deliver. +func postHeartbeatToMirror(ctx context.Context, cfg mirrorHubConfig, payload *HeartbeatPayload, logger *slog.Logger) { + recordMirrorAttempt() + body, err := json.Marshal(payload) + if err != nil { + recordMirrorError("marshal failed: " + err.Error()) + return + } + reqCtx, cancel := context.WithTimeout(ctx, cfg.timeout) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, cfg.url+mirrorHeartbeatPath, bytes.NewReader(body)) + if err != nil { + recordMirrorError("request failed: " + err.Error()) + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+cfg.key) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + msg := "unreachable: " + mirrorTransportError(err) + recordMirrorError(msg) + if logger != nil { + logger.Debug("mirror hub heartbeat failed", "mirror_host", req.URL.Host, "error", mirrorTransportError(err)) + } + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, mirrorErrorBodyMaxBytes)) + msg := fmt.Sprintf("rejected: HTTP %d %s", resp.StatusCode, strings.TrimSpace(string(respBody))) + recordMirrorError(msg) + if logger != nil { + logger.Debug("mirror hub heartbeat rejected", "mirror_host", req.URL.Host, "status", resp.StatusCode) + } + return + } + recordMirrorSuccess() + + raw, err := io.ReadAll(io.LimitReader(resp.Body, mirrorResponseMaxBytes)) + if err != nil { + return + } + ignored, err := mirrorResponseAuthorityFields(raw) + if err != nil || len(ignored) == 0 { + return + } + noteMirrorIgnoredFields(ignored, req.URL.Host, logger) +} + +// noteMirrorIgnoredFields counts authority fields a mirror response carried and +// logs mirrorResponseIgnoredEvent at most once per mirrorIgnoredLogWindow. +// Names only: the values (which may be credentials) are never logged. +func noteMirrorIgnoredFields(ignored []string, host string, logger *slog.Logger) { + mirrorIgnoredTotal.Add(int64(len(ignored))) + now := time.Now() + mirrorState.Lock() + mirrorState.lastIgnoredFields = append([]string(nil), ignored...) + shouldLog := mirrorState.ignoredLoggedAt.IsZero() || now.Sub(mirrorState.ignoredLoggedAt) >= mirrorIgnoredLogWindow + if shouldLog { + mirrorState.ignoredLoggedAt = now + } + mirrorState.Unlock() + if shouldLog && logger != nil { + logger.Warn(mirrorResponseIgnoredEvent+": the mirror hub's heartbeat response carried config, credential or upgrade fields; a report-only mirror has no authority over this hive and they were not applied", + "event", mirrorResponseIgnoredEvent, + "mirror_host", host, + "fields", strings.Join(ignored, ","), + "ignored_total", mirrorIgnoredTotal.Load(), + ) + } +} + +// mirrorTransportError reduces a transport error to a short, URL-free form so +// the status block and logs never echo a query string or userinfo. +func mirrorTransportError(err error) string { + var uerr *url.Error + if errors.As(err, &uerr) && uerr.Err != nil { + if errors.Is(uerr.Err, context.DeadlineExceeded) { + return "timeout" + } + return uerr.Err.Error() + } + if errors.Is(err, context.DeadlineExceeded) { + return "timeout" + } + return err.Error() +} diff --git a/src/pkg/hub/spoke/heartbeat_mirror_test.go b/src/pkg/hub/spoke/heartbeat_mirror_test.go new file mode 100644 index 0000000000..a24331348d --- /dev/null +++ b/src/pkg/hub/spoke/heartbeat_mirror_test.go @@ -0,0 +1,581 @@ +package spoke + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" +) + +const ( + mirrorTestPrimaryKey = "primary-key-must-stay-with-primary" + mirrorTestMirrorKey = "mirror-key-never-logged-0123456789" + mirrorTestWait = 5 * time.Second +) + +// TestHeartbeatMirrorFieldPolicyInventory fails when a HeartbeatPayload field +// has no mirror classification, or when the policy names a field that no +// longer exists. A new field must be classified before it can ship (#18148). +func TestHeartbeatMirrorFieldPolicyInventory(t *testing.T) { + typ := reflect.TypeOf(HeartbeatPayload{}) + seen := map[string]bool{} + var missing []string + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + seen[name] = true + switch heartbeatMirrorFieldPolicy[name] { + case mirrorFieldMirrored, mirrorFieldDropped: + default: + missing = append(missing, name) + } + } + if len(missing) > 0 { + sort.Strings(missing) + t.Errorf("HeartbeatPayload fields with no mirror classification: %v\n"+ + "add each to heartbeatMirrorFieldPolicy in heartbeat_mirror.go as "+ + "mirrorFieldMirrored or mirrorFieldDropped (credential-adjacent identifier, "+ + "authentication error text, signed-in users, activity digest, primary-only consent)", missing) + } + var stale []string + for name := range heartbeatMirrorFieldPolicy { + if !seen[name] { + stale = append(stale, name) + } + } + if len(stale) > 0 { + sort.Strings(stale) + t.Errorf("heartbeatMirrorFieldPolicy names fields HeartbeatPayload no longer has: %v", stale) + } +} + +// The fields the RFC names explicitly must stay dropped. +func TestHeartbeatMirrorFieldPolicyDropsRFCFields(t *testing.T) { + for _, name := range []string{ + "DashboardTokenHash", "GitHubAppKeyFingerprint", "WebhookRelayKeyFingerprint", "GitHubAppKeysHeld", + "InferenceAuthError", "GitHubAppTokenError", + "ActiveSessionUsers", "EngagedSessionUsers", "UserLastActions", + "Activity", + } { + if heartbeatMirrorFieldPolicy[name] != mirrorFieldDropped { + t.Errorf("%s must be mirrorFieldDropped", name) + } + } +} + +// Every name in the harmless table must exist on HeartbeatResponse, and the +// authority fields the hub fences for self-service hives must never be listed +// as harmless here. +func TestHeartbeatMirrorResponseHarmlessInventory(t *testing.T) { + typ := reflect.TypeOf(HeartbeatResponse{}) + exists := map[string]bool{} + for i := 0; i < typ.NumField(); i++ { + exists[typ.Field(i).Name] = true + } + for name := range heartbeatMirrorResponseHarmless { + if !exists[name] { + t.Errorf("heartbeatMirrorResponseHarmless names %s, which HeartbeatResponse no longer has", name) + } + } + for _, name := range []string{ + "GitHubAppConfig", "PendingGateway", "ProjectConfig", "ClusterID", + "UpgradeTo", "UpgradeInterrupt", "SwitchToTag", "UpgradeAnnounce", "UpgradePolicy", + "RestartSpoke", "ResetAgentRestarts", "Breaker", "SelfExperimentFlag", + "IsPublic", "AuthorizedUsers", "AuthorizedUserNames", "HubBanner", + } { + if heartbeatMirrorResponseHarmless[name] { + t.Errorf("%s carries authority and must not be harmless for a mirror", name) + } + } +} + +// filledHeartbeatPayload sets every settable field of a HeartbeatPayload to a +// non-zero value, so the reducer test covers fields added after it was written. +func filledHeartbeatPayload(t *testing.T) *HeartbeatPayload { + t.Helper() + p := &HeartbeatPayload{} + v := reflect.ValueOf(p).Elem() + for i := 0; i < v.NumField(); i++ { + fv := v.Field(i) + if !fv.CanSet() { + continue + } + mirrorFillNonZero(t, v.Type().Field(i).Name, fv) + if fv.IsZero() { + t.Fatalf("could not give HeartbeatPayload.%s a non-zero value", v.Type().Field(i).Name) + } + } + return p +} + +func mirrorFillNonZero(t *testing.T, name string, fv reflect.Value) { + t.Helper() + switch fv.Kind() { + case reflect.Bool: + fv.SetBool(true) + case reflect.String: + fv.SetString("x") + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + fv.SetInt(1) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + fv.SetUint(1) + case reflect.Float32, reflect.Float64: + fv.SetFloat(1) + case reflect.Slice: + fv.Set(reflect.MakeSlice(fv.Type(), 1, 1)) + case reflect.Map: + fv.Set(reflect.MakeMap(fv.Type())) + key := reflect.New(fv.Type().Key()).Elem() + if key.Kind() == reflect.String { + key.SetString("k") + } + fv.SetMapIndex(key, reflect.Zero(fv.Type().Elem())) + case reflect.Pointer: + fv.Set(reflect.New(fv.Type().Elem())) + case reflect.Struct: + for i := 0; i < fv.NumField(); i++ { + if fv.Field(i).CanSet() { + mirrorFillNonZero(t, name, fv.Field(i)) + return + } + } + default: + t.Fatalf("mirrorFillNonZero: unhandled kind %s for HeartbeatPayload.%s", fv.Kind(), name) + } +} + +func TestReduceHeartbeatForMirrorZeroesDroppedFieldsOnly(t *testing.T) { + src := filledHeartbeatPayload(t) + before := clonePayload(src) + + reduced := reduceHeartbeatForMirror(src, "") + if reduced == nil { + t.Fatal("reduced payload is nil") + } + v := reflect.ValueOf(reduced).Elem() + for i := 0; i < v.NumField(); i++ { + name := v.Type().Field(i).Name + switch heartbeatMirrorFieldPolicy[name] { + case mirrorFieldMirrored: + if v.Field(i).IsZero() { + t.Errorf("mirrored field %s was cleared", name) + } + default: + if !v.Field(i).IsZero() { + t.Errorf("dropped field %s still set in the mirror copy", name) + } + } + } + if !reflect.DeepEqual(clonePayload(src), before) { + t.Error("the caller's payload was modified by the reducer") + } + + if got := reduceHeartbeatForMirror(src, "byo-alice-lab"); got.HiveID != "byo-alice-lab" { + t.Errorf("hive id override = %q, want byo-alice-lab", got.HiveID) + } + if got := reduceHeartbeatForMirror(src, ""); got.HiveID != src.HiveID { + t.Errorf("hive id without override = %q, want %q", got.HiveID, src.HiveID) + } + if reduceHeartbeatForMirror(nil, "") != nil { + t.Error("nil payload must reduce to nil") + } +} + +func TestLoadMirrorHubConfig(t *testing.T) { + t.Setenv(EnvMirrorHubHeartbeatKey, mirrorTestMirrorKey) + + t.Setenv(EnvMirrorHubURL, "https://mirror.example.test/") + cfg, err := loadMirrorHubConfig("https://hub.example.test") + if err != nil { + t.Fatalf("valid config rejected: %v", err) + } + if cfg.url != "https://mirror.example.test" || cfg.key != mirrorTestMirrorKey || cfg.timeout != defaultMirrorHubTimeout { + t.Errorf("cfg = %+v", mirrorHubConfig{url: cfg.url, hiveID: cfg.hiveID, timeout: cfg.timeout}) + } + + t.Setenv(EnvMirrorHubTimeout, "250ms") + if cfg, err = loadMirrorHubConfig(""); err != nil || cfg.timeout != 250*time.Millisecond { + t.Errorf("timeout override: cfg.timeout=%v err=%v", cfg.timeout, err) + } + t.Setenv(EnvMirrorHubTimeout, "nonsense") + if cfg, err = loadMirrorHubConfig(""); err != nil || cfg.timeout != defaultMirrorHubTimeout { + t.Errorf("invalid timeout must keep the default: cfg.timeout=%v err=%v", cfg.timeout, err) + } + + if _, err := loadMirrorHubConfig("https://mirror.example.test"); err == nil { + t.Error("mirror equal to the primary hub must be rejected") + } + t.Setenv(EnvMirrorHubURL, "mirror.example.test") + if _, err := loadMirrorHubConfig(""); err == nil { + t.Error("URL without a scheme must be rejected") + } + t.Setenv(EnvMirrorHubURL, "https://mirror.example.test") + t.Setenv(EnvMirrorHubHeartbeatKey, "") + _, err = loadMirrorHubConfig("") + if err == nil { + t.Fatal("missing mirror key must be rejected") + } + if !strings.Contains(err.Error(), EnvMirrorHubHeartbeatKey) || strings.Contains(err.Error(), mirrorTestMirrorKey) { + t.Errorf("error should name the variable and never a key: %v", err) + } +} + +// mirrorLogSink is a goroutine-safe slog destination. +type mirrorLogSink struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (s *mirrorLogSink) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Write(p) +} + +func (s *mirrorLogSink) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.String() +} + +// capturedBeat is one request a fake hub received. +type capturedBeat struct { + auth string + body map[string]any +} + +// fakeHub is an httptest hub that records every heartbeat and answers with +// the configured response body. +type fakeHub struct { + srv *httptest.Server + hits atomic.Int64 + mu sync.Mutex + beats []capturedBeat +} + +func newFakeHub(t *testing.T, status int, response any) *fakeHub { + t.Helper() + h := &fakeHub{} + h.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + h.mu.Lock() + h.beats = append(h.beats, capturedBeat{auth: r.Header.Get("Authorization"), body: body}) + h.mu.Unlock() + h.hits.Add(1) + w.WriteHeader(status) + if response != nil { + _ = json.NewEncoder(w).Encode(response) + } + })) + t.Cleanup(h.srv.Close) + return h +} + +func (h *fakeHub) last(t *testing.T) capturedBeat { + t.Helper() + h.mu.Lock() + defer h.mu.Unlock() + if len(h.beats) == 0 { + t.Fatal("hub received no beat") + } + return h.beats[len(h.beats)-1] +} + +func (h *fakeHub) host(t *testing.T) string { + t.Helper() + u, err := url.Parse(h.srv.URL) + if err != nil { + t.Fatal(err) + } + return u.Host +} + +// setMirrorEnv points the mirror at srv with the test mirror key and the +// primary at the test primary key. +func setMirrorEnv(t *testing.T, mirrorURL string) { + t.Helper() + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + t.Setenv(EnvHeartbeatKey, mirrorTestPrimaryKey) + t.Setenv("HIVE_HUB_SECRET", "") + t.Setenv(EnvMirrorHubURL, mirrorURL) + t.Setenv(EnvMirrorHubHeartbeatKey, mirrorTestMirrorKey) + t.Setenv(EnvMirrorHubHiveID, "") + t.Setenv(EnvMirrorHubTimeout, "") +} + +func mirrorTestPayload() *HeartbeatPayload { + return &HeartbeatPayload{ + HiveID: "hive-primary", + Org: "acme", + Repos: []string{"acme/one", "acme/two"}, + PrimaryRepo: "acme/one", + ACMMLevel: 4, + Agents: []AgentSummary{{Name: "reviewer", State: "running"}}, + DashboardTokenHash: "deadbeef", + GitHubAppKeyFingerprint: "sha256:aaaa", + InferenceAuthError: "401 from gateway", + ActiveSessionUsers: []string{"alice"}, + UserLastActions: map[string]string{"alice": "2026-01-01T00:00:00Z"}, + Version: "v5", + } +} + +// droppedJSONNames returns the JSON keys of every dropped HeartbeatPayload field. +func droppedJSONNames() map[string]bool { + out := map[string]bool{} + typ := reflect.TypeOf(HeartbeatPayload{}) + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + if heartbeatMirrorFieldPolicy[f.Name] == mirrorFieldMirrored { + continue + } + out[strings.Split(f.Tag.Get("json"), ",")[0]] = true + } + return out +} + +func TestMirrorHeartbeat_SendsReducedCopyWithMirrorKey(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + mirror := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + setMirrorEnv(t, mirror.srv.URL) + t.Setenv(EnvMirrorHubHiveID, "byo-alice-hive") + + resp := postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), nil2Logger()) + if resp == nil || !resp.OK { + t.Fatalf("primary response = %+v, want ok", resp) + } + testutil.Eventually(t, mirrorTestWait, func() bool { return mirror.hits.Load() == 1 }, "mirror hub never received the beat") + + p, m := primary.last(t), mirror.last(t) + if p.auth != "Bearer "+mirrorTestPrimaryKey { + t.Errorf("primary auth = %q, want the primary key", p.auth) + } + if m.auth != "Bearer "+mirrorTestMirrorKey { + t.Errorf("mirror auth = %q, want the mirror key", m.auth) + } + for key := range droppedJSONNames() { + if _, present := m.body[key]; present { + t.Errorf("mirror copy still carries dropped field %q", key) + } + delete(p.body, key) + } + if m.body["hive_id"] != "byo-alice-hive" { + t.Errorf("mirror hive_id = %v, want the override", m.body["hive_id"]) + } + delete(p.body, "hive_id") + delete(m.body, "hive_id") + if !reflect.DeepEqual(p.body, m.body) { + t.Errorf("mirror copy differs from the primary beat beyond the drop list and hive_id:\nprimary: %v\nmirror: %v", p.body, m.body) + } + + st, ok := MirrorHubStatusSnapshot() + if !ok { + t.Fatal("mirror status should be present when configured") + } + if st.URLHost != mirror.host(t) || st.LastOK == "" || st.LastError != "" || st.IgnoredFields != 0 { + t.Errorf("mirror status = %+v", st) + } +} + +func TestMirrorHeartbeat_IgnoresAuthorityFieldsAndLogsOncePerWindow(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + mirrorResp := map[string]any{ + "ok": true, + "upgrade_to": "abc123", + "switch_to_tag": "v6", + "restart_spoke": true, + "breaker": BreakerEngage, + "is_public": true, + "cluster_id": "mirror-cluster", + "authorized_users": []string{"mallory:admin"}, + "github_app_config": map[string]any{"app_id": 42, "installation_id": 7, "private_key": "-----BEGIN RSA PRIVATE KEY-----"}, + "pending_gateway": map[string]any{"name": "openrouter", "key": "sk-mirror-secret"}, + "project_config": map[string]any{"org": "mallory"}, + "upgrade_policy": map[string]any{"hub_managed": true}, + } + mirror := newFakeHub(t, http.StatusOK, mirrorResp) + setMirrorEnv(t, mirror.srv.URL) + saved := mirrorIgnoredLogWindow + mirrorIgnoredLogWindow = time.Hour + t.Cleanup(func() { mirrorIgnoredLogWindow = saved }) + + sink := &mirrorLogSink{} + logger := slog.New(slog.NewTextHandler(sink, &slog.HandlerOptions{Level: slog.LevelDebug})) + + const beats = 2 + const authorityFieldsPerResponse = 11 + for i := 0; i < beats; i++ { + resp := postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), logger) + if resp == nil || !resp.OK || resp.UpgradeTo != "" || resp.RestartSpoke || resp.GitHubAppConfig != nil || resp.PendingGateway != nil { + t.Fatalf("primary response contaminated by the mirror: %+v", resp) + } + testutil.Eventually(t, mirrorTestWait, func() bool { return mirror.hits.Load() == int64(i+1) }, "mirror beat %d never arrived", i+1) + testutil.Eventually(t, mirrorTestWait, func() bool { + st, _ := MirrorHubStatusSnapshot() + return st.IgnoredFields == int64(authorityFieldsPerResponse*(i+1)) + }, "ignored fields not counted for beat %d", i+1) + } + + st, _ := MirrorHubStatusSnapshot() + sort.Strings(st.LastIgnoredFields) + want := []string{"authorized_users", "breaker", "cluster_id", "github_app_config", "is_public", "pending_gateway", "project_config", "restart_spoke", "switch_to_tag", "upgrade_policy", "upgrade_to"} + if !reflect.DeepEqual(st.LastIgnoredFields, want) { + t.Errorf("last ignored fields = %v, want %v", st.LastIgnoredFields, want) + } + + logs := sink.String() + if n := strings.Count(logs, "event="+mirrorResponseIgnoredEvent); n != 1 { + t.Errorf("%s logged %d times across %d beats, want once per window:\n%s", mirrorResponseIgnoredEvent, n, beats, logs) + } + for _, secret := range []string{mirrorTestMirrorKey, mirrorTestPrimaryKey, "sk-mirror-secret", "BEGIN RSA"} { + if strings.Contains(logs, secret) { + t.Errorf("logs contain %q:\n%s", secret, logs) + } + } + if !strings.Contains(logs, "mirror_host="+mirror.host(t)) { + t.Errorf("log should name the mirror host:\n%s", logs) + } + if _, ok := LastHeartbeatSuccess(); !ok { + t.Error("primary success must be recorded") + } +} + +func TestMirrorHeartbeat_RejectionDoesNotAffectPrimary(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + mirror := newFakeHub(t, http.StatusInternalServerError, nil) + setMirrorEnv(t, mirror.srv.URL) + sink := &mirrorLogSink{} + logger := slog.New(slog.NewTextHandler(sink, &slog.HandlerOptions{Level: slog.LevelDebug})) + + resp := postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), logger) + if resp == nil || !resp.OK { + t.Fatalf("primary response = %+v, want ok despite the mirror 500", resp) + } + if _, ok := LastHeartbeatSuccess(); !ok { + t.Error("primary success must be recorded") + } + testutil.Eventually(t, mirrorTestWait, func() bool { + st, _ := MirrorHubStatusSnapshot() + return strings.Contains(st.LastError, "HTTP 500") + }, "mirror rejection not surfaced in status") + st, _ := MirrorHubStatusSnapshot() + if st.LastOK != "" || st.LastAttempt == "" { + t.Errorf("mirror status = %+v", st) + } + if strings.Contains(sink.String(), mirrorTestMirrorKey) { + t.Error("mirror key leaked into logs") + } +} + +func TestMirrorHeartbeat_TimeoutDoesNotDelayPrimary(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + release := make(chan struct{}) + mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + // Cleanups run last-registered first: release the handler, then close. + t.Cleanup(mirror.Close) + t.Cleanup(func() { close(release) }) + setMirrorEnv(t, mirror.URL) + t.Setenv(EnvMirrorHubTimeout, "50ms") + + // The mirror handler blocks until cleanup, so this call can only return if + // the mirror POST is decoupled from the primary beat. + resp := postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), nil2Logger()) + if resp == nil || !resp.OK { + t.Fatalf("primary response = %+v, want ok while the mirror hangs", resp) + } + testutil.Eventually(t, mirrorTestWait, func() bool { + st, _ := MirrorHubStatusSnapshot() + return st.LastError == "unreachable: timeout" + }, "mirror timeout not surfaced in status") + testutil.Eventually(t, mirrorTestWait, func() bool { return !mirrorInFlight.Load() }, "mirror goroutine did not finish after its timeout") +} + +func TestMirrorHeartbeat_SkipsWhileAnotherBeatInFlight(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + mirror := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + setMirrorEnv(t, mirror.srv.URL) + + // Simulate a mirror POST that has not returned yet. + mirrorInFlight.Store(true) + postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), nil2Logger()) + st, _ := MirrorHubStatusSnapshot() + if !strings.Contains(st.LastError, "in flight") { + t.Errorf("skipped beat not recorded: %+v", st) + } + if mirror.hits.Load() != 0 { + t.Error("a second mirror POST was started while one was in flight") + } + mirrorInFlight.Store(false) +} + +func TestMirrorHeartbeat_WithoutMirrorKeySendsNothing(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + mirror := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + setMirrorEnv(t, mirror.srv.URL) + t.Setenv(EnvMirrorHubHeartbeatKey, "") + + resp := postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), nil2Logger()) + if resp == nil || !resp.OK { + t.Fatalf("primary response = %+v, want ok", resp) + } + // No goroutine is started on a configuration error, so the status is + // final as soon as postHeartbeatToHub returns. + st, ok := MirrorHubStatusSnapshot() + if !ok || !strings.Contains(st.LastError, EnvMirrorHubHeartbeatKey) { + t.Errorf("mirror status = %+v, want a configuration error naming %s", st, EnvMirrorHubHeartbeatKey) + } + if mirror.hits.Load() != 0 { + t.Error("the primary key must never be offered to the mirror hub: a beat was sent without a mirror key") + } + if primary.last(t).auth != "Bearer "+mirrorTestPrimaryKey { + t.Error("primary auth changed") + } +} + +func TestMirrorHeartbeat_UnsetIsANoop(t *testing.T) { + primary := newFakeHub(t, http.StatusOK, HeartbeatResponse{OK: true}) + setMirrorEnv(t, "") + + if resp := postHeartbeatToHub(context.Background(), primary.srv.URL, mirrorTestPayload(), nil2Logger()); resp == nil || !resp.OK { + t.Fatalf("primary response = %+v, want ok", resp) + } + if st, ok := MirrorHubStatusSnapshot(); ok { + t.Errorf("mirror status present with %s unset: %+v", EnvMirrorHubURL, st) + } + if mirrorInFlight.Load() { + t.Error("a mirror goroutine was started with no mirror configured") + } +} + +func TestMirrorResponseAuthorityFields(t *testing.T) { + names, err := mirrorResponseAuthorityFields([]byte(`{"ok":true,"hub_git_hash":"h","latest_sha":"s","sig_seq":3,"activity_etag":"e"}`)) + if err != nil || len(names) != 0 { + t.Errorf("harmless response = %v, %v; want no authority fields", names, err) + } + names, err = mirrorResponseAuthorityFields([]byte(`{"ok":true,"upgrade_to":"x","hub_banner":{"id":"b","message":"m"}}`)) + if err != nil { + t.Fatal(err) + } + sort.Strings(names) + if !reflect.DeepEqual(names, []string{"hub_banner", "upgrade_to"}) { + t.Errorf("authority fields = %v", names) + } + if _, err := mirrorResponseAuthorityFields([]byte("not json")); err == nil { + t.Error("malformed response must report an error, not a count") + } +} diff --git a/src/pkg/hub/spoke/heartbeat_testhooks.go b/src/pkg/hub/spoke/heartbeat_testhooks.go index 48743a0619..7af4b161e6 100644 --- a/src/pkg/hub/spoke/heartbeat_testhooks.go +++ b/src/pkg/hub/spoke/heartbeat_testhooks.go @@ -42,4 +42,6 @@ func ResetHeartbeatStateForTest() { heartbeatIdentity.Lock() heartbeatIdentity.payload = nil heartbeatIdentity.Unlock() + // The report-only mirror's bookkeeping (#18148) is process-global too. + resetMirrorStateForTest() } From 155641be487eb2583961db45d37c51c4b39dde04 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 22:04:43 -0500 Subject: [PATCH 206/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20reference=20repo?= =?UTF-8?q?s=20derived=20from=20an=20accepted=20request=20issue,=20operato?= =?UTF-8?q?r=20reference,=20intake=20hook=20and=20TechDocs=20walkthrough?= =?UTF-8?q?=20(#18139=20step=206)=20(#18177)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents the feature that landed in #18163, #18171, #18169 and #18174: the derived-entry flow and credential fallback in agent-configuration.md, new notable-field rows in the operator reference, the spec.intake hook in the workflow-models design doc, an operator walkthrough for the Hive Commons pilot in techdocs-assessment.md, and the feature-level changelog fragment. Also corrects the App-installation sentence: the hive owner cannot install the App on a third-party org; the installation must already exist there. Fixes #18162 Part of #18139 Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...d-18139-derived-reference-repos-feature.md | 1 + src/docs/agent-configuration.md | 18 ++++++++- src/docs/design/workflow-models.md | 39 +++++++++++++++++++ src/docs/operator-reference/README.md | 9 ++++- src/docs/workflows/techdocs-assessment.md | 33 ++++++++++++++++ 5 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-18139-derived-reference-repos-feature.md diff --git a/changelog.d/added-18139-derived-reference-repos-feature.md b/changelog.d/added-18139-derived-reference-repos-feature.md new file mode 100644 index 0000000000..45083826d0 --- /dev/null +++ b/changelog.d/added-18139-derived-reference-repos-feature.md @@ -0,0 +1 @@ +- A hive now derives write-tier `project.reference_repos` entries from an accepted request issue instead of needing the operator to hand-write them ([#18139](https://github.com/hivecommons/hive/issues/18139)). When an authorized repo's bound workflow model declares `intake.referenceRepoFields` (first: `cncf-techdocs-assessment@curated-2026.11.2`; hives bound to 11.1 must rebind), every repository the request form names in scope becomes an `issues+prs` reference repo once a person with write access applies `triage/accepted`; a bot applying the label grants nothing. The credential is the App installation on the requesting org, with a read-only fallback and one marked comment on the request issue when there is none, so a derived entry never fails boot. Entries are rebuilt at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`), vanish when the issue closes or loses the label, are never written to any saved config, yield to a hand-written entry on the same repo, carry `(from owner/repo#N)` in the kick, and appear read-only in Settings with their source and credential state. Docs: `agent-configuration.md`, `operator-reference/README.md`, `design/workflow-models.md`, `workflows/techdocs-assessment.md` (#18157, #18158, #18159, #18160, #18161, #18162). diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 9b909897ae..7d1d116163 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1181,10 +1181,26 @@ project: | `read-only` (string form, or no `write`) | `git clone`/fetch over HTTPS, GitHub REST/GraphQL GETs. Every request goes out without auth because the hive's App token is removed, so a private reference repo is not readable. | Every non-GET, with a 403 that names `reference_repos`. | | `issues`, `prs` or `issues+prs` (`write:`) | The reads above, plus only: open an issue (`issues`), open a PR or push a topic branch (`prs`), and comment on an issue or PR the hive itself opened. | Merges, pushes to the default branch, labels, assignees, reviews, and closing other people's items. Each refusal is logged as `reference_repo` in the violations log, so retrying cannot succeed. | -Writes on the opt-in tier use a credential valid for *that* repo, never the hive's own installation token. By default the same GitHub App must be installed on the target org: the hive discovers that installation and mints a token limited to the one repo and the opted-in permissions. Otherwise, set `write_token_file`. If a write-mode entry has neither, boot fails with exit code 22. These are direct POSTs. The `hive-open-issue`/`hive-open-pr` relays only act on `project.repos`, so a lane under `write_surface.enforce` cannot write to a reference repo at all ([GitHub write surface](github-write-surface.md#reference-repos)). +Writes on the opt-in tier use a credential valid for *that* repo, never the hive's own installation token. The credential comes from an installation of the same GitHub App on the target org: the hive discovers that installation and mints a token limited to the one repo and the opted-in permissions. The hive owner cannot install the App on a third-party org. The installation must already exist there, which it does whenever that org runs its own hive (every hivecommons repo has the App installed, for example). For a hand-written entry whose org has no installation, `write_token_file` is the fallback. If a hand-written write-mode entry has neither, boot fails with exit code 22. These are direct POSTs. The `hive-open-issue`/`hive-open-pr` relays only act on `project.repos`, so a lane under `write_surface.enforce` cannot write to a reference repo at all ([GitHub write surface](github-write-surface.md#reference-repos)). What agents see: every kick carries a `📚 REFERENCE REPOS` block listing each repo with its tier (`[reference · read-only]`, `[reference · issues+prs]`, …). It tells agents that an authorized-repo item whose deliverables name a reference repo is **in scope**. Findings belong in the deliverable inside the authorized repo, unless the tier allows an issue or PR. On the dashboard, each reference repo is a minimal card with a `reference · <tier>` chip and no stats, ACMM level or work items. Reference repos come in a separate payload field, so no queue or ACMM consumer counts them. Field-by-field defaults are in the [operator reference](operator-reference/README.md#notable-fields). +#### Reference repos derived from an accepted request + +Hand-writing `reference_repos` does not scale when the hive's work arrives as requests. A TechDocs hive receives one assessment request per project, and each request names the repositories it is about. The hive therefore derives write-tier entries from an accepted request issue ([#18139](https://github.com/hivecommons/hive/issues/18139)). Nothing in this flow needs the operator to edit the config. + +The flow: + +1. A request issue is opened in an authorized repo (`cncf/techdocs#395`, say) through an issue form. Its body carries the form headings and the repositories under them. +2. The workflow model bound to that repo declares which headings matter: `spec.intake.referenceRepoFields` lists the headings whose repositories are in scope and `spec.intake.outOfScopeFields` the headings whose repositories are excluded. A model without `intake.referenceRepoFields` derives nothing. For TechDocs that is `cncf-techdocs-assessment@curated-2026.11.2`; `curated-2026.11.1` is unchanged, so a hive bound to it must rebind. +3. A person with write access to the authorized repo applies `triage/accepted`. That label is the grant. The hive reads who applied it from the issue timeline and checks their repository permission. A label applied by a bot or App account grants nothing, so the hive can never grant itself access by moving the label. A label applied by someone without write access grants nothing either. +4. Every repository under an in-scope heading, and not under an out-of-scope heading, becomes a `write: [issues, prs]` entry. The issue's own repo, any authorized repo and any repo in `project.repos` are skipped. +5. The credential is the App installation on the requesting org, exactly as for a hand-written write-tier entry. When the org has no installation, the entry is served read-only instead, the hive keeps running, and the request issue gets exactly one comment marked `<!-- hive:reference-repos-read-only repos=... state=read-only -->` saying which repos the hive can read but not write and how an owner of that org can change that (install the same App; the hive picks it up on the next reload). The comment is posted once per request issue and repo set, never again on a plain reload. A derived entry never consults `write_token_file` and never fails boot. + +Lifecycle: the hive derives at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`, Go duration, see [env-vars.md](env-vars.md)). Closing the request issue or removing `triage/accepted` drops the entry, its proxy policy and its kick line on the next pass. A hand-written entry for the same repo always wins over the derived one. Derived entries live in memory only: a saved overlay or runtime config never contains one, and a restart rebuilds them from the open issues. The same repo named by two accepted issues keeps the first and logs the second. + +What agents see: the `📚 REFERENCE REPOS` block appends `(from owner/repo#N)` to each derived line, and `(from owner/repo#N; read-only: <reason>)` when the credential fell back. The proxy rules are the ones in the table above: a derived entry unlocks opening issues and PRs on that repo and nothing else, and the read-only fallback unlocks nothing. On the dashboard, Settings shows derived entries read-only with a link to the request issue and the credential state, and the API answers an edit of one with `409`. + ## Label policy: which issues agents may work There is exactly **one** label-policy surface for the hive's own agents — the **Governor Configuration → Labels** tab — and it has two polarities: diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index cba8a0dd33..de65c61ef7 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -522,6 +522,45 @@ fixture under `pkg/workflowmodel/ingest/testdata/techdocs` carries both. - Branch protection needs `administration:read`; without it the rulesets endpoint is used, which omits legacy protection settings such as `enforce_admins`. +### Intake hook: reference repos from a request issue + +A model may declare, in `spec.intake`, which issue-form headings name repositories the work is +*about* ([#18139](https://github.com/hivecommons/hive/issues/18139)). This is a model capability, +not a hive setting: binding a model that declares it is what turns derivation on for that repo. + +```yaml +spec: + intake: + referenceRepoFields: # headings whose values name repositories in scope + - "Website or documentation repository" + - "Other repositories or sites in scope" + outOfScopeFields: # headings whose repositories are excluded + - "Out of scope" +``` + +`Intake{ReferenceRepoFields, OutOfScopeFields}` (`json:"intake,omitzero"`, `pkg/workflowmodel/model.go`) +is omitted on every existing model, so stored hashes are unchanged and the strict curated decoder +keeps accepting the current files; setting it changes the spec hash. The entries are the rendered +`### <label>` headings as they appear in a request issue body, not the form's field ids, because the +hive only ever sees the body. CamelCase follows the rest of the model (`guardRef`, `workTypes`). + +The pure parser lives in `pkg/refderive`: `ParseIssueForm` reads the body into heading/value +fields, `ReposInText` extracts `owner/repo` from URLs on the hive's host anywhere in the text and +from bare slugs only when the slug is a whole line or list item, and `Derive` returns one +`issues+prs` `config.ReferenceRepo` per in-scope repo with `DerivedFrom: owner/repo#N`. It refuses +with a typed error when the issue is closed, lacks `triage/accepted`, the label's applier is +unknown, lacks write access or is a bot, or the model declares no intake fields. Everything that +talks to GitHub (listing candidates, reading the label applier from the timeline, resolving the +credential) is in `cmd/hive` and `pkg/github`; the operator view is in +[agent-configuration.md](../agent-configuration.md#reference-repos-derived-from-an-accepted-request). + +TechDocs is the first model to declare the hook: `cncf-techdocs-assessment@curated-2026.11.2` +([techdocs-assessment.md](../workflows/techdocs-assessment.md#reference-repos-from-the-request)). +Curated files are never edited in place, so `curated-2026.11.1` stays without the block and a hive +bound to it derives nothing. The fixture `pkg/refderive/testdata/techdocs-395.md` (trimmed +`cncf/techdocs#395`) yields `hivecommons/docs`, `hivecommons/hivecommons.github.io` and +`hivecommons/hive`. + ## Label GC and model commands (RFC phase 4, follow-up) Two of the phase-4 backend gaps close here: stale model labels are now reclaimed, and a bound diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 7c0ce4320d..b4f7a7b99c 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -252,8 +252,13 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `project.repo_policies[].label_driven_approval_labels` | Unset: inherits `project.label_driven_approval_labels`. | Per-repo approval-label list; only read when `label_driven` is set. Editable from the dashboard sub-row of the repo's label-driven toggle. | | `project.repo_policies[].companion_repos` | Unset (no companions). | List of sibling repositories (`name` in the project org, or `org/name`) that this repo's tooling expects checked out beside it under the agent workspace root — for example a docs/website repo a spec tool registers at `<workspace>/<org>/<sibling>/…`. Contributor task prompts and resident-agent kicks carry a shallow-clone plan for each companion at `<workspace root>/<org>/<sibling>` (`$HIVE_WORKSPACE_DIR` for contributor agents, `$HOME/workspace` for resident agents), and agents are told that a tool reporting one of those paths missing means *clone it*, never park the issue. Self-references and malformed entries are dropped. See [running-at-level-6.md](../running-at-level-6.md#needless-parks). | | `project.reference_repos` | Unset (none). | Repos **outside** the project the hive may *study* without managing them ([#18015](https://github.com/hivecommons/hive/issues/18015)) — the rare case of a hive whose deliverable is *about* another project, e.g. a CNCF TechDocs hive (authorized repo `cncf/techdocs`) assessing `hivecommons/docs`. Each entry is `owner/name` (bare names are rejected, as is any repo also in `project.repos`). Default tier is **read-only**: agents may `git clone`/fetch them unauthenticated over HTTPS and issue GitHub REST/GraphQL GETs; the proxy strips the hive's App token from every request to them and refuses every non-GET with a message naming `reference_repos`. The kick lists them under `📚 REFERENCE REPOS` so an authorized-repo item whose deliverables name one is treated as in scope (not out-of-scope/critical), and agents are told findings go in the deliverable inside the authorized repo, never as issues or PRs on the reference repo. The dashboard shows each as a minimal card with a `reference · <tier>` chip — no stats, no ACMM, no work items. Private reference repos are not readable (reads are unauthenticated). Example: <br>`project:`<br>` repos: [techdocs]`<br>` reference_repos:`<br>` - hivecommons/docs`<br>` - hivecommons/hivecommons.github.io`<br>` - {repo: hivecommons/hive, write: [issues, prs]}` | -| `project.reference_repos[].write` | Unset (read-only). | Per-entry write opt-in, any of `issues`, `prs`. The **only** writes it unlocks on that repo: open an issue; open a PR / push a topic branch; comment on an issue or PR the hive itself opened. Never: merge, push to the default branch, labels or assignees on others' items, reviews, closing others' issues — the proxy refuses those deterministically (`reference_repo` in the violations log). The write is sent with a credential GitHub honours for *that* repo, never the hive's own installation token: by default the same GitHub App must be **installed on the target org** (the hive discovers that installation and mints a token scoped to the one repo and the opted-in permissions); otherwise set `write_token_file`. Boot refuses (exit code 22) a write-mode entry with neither. Direct POSTs are used here, not the `hive-open-pr`/`hive-open-issue` relays, which only act on `project.repos`; a lane under `write_surface.enforce` therefore cannot write to a reference repo. | -| `project.reference_repos[].write_token_file` | Unset. | Path to a file holding a token (a maintainer PAT or fine-grained token) authorized for that repo; read on every use so a rotation needs no restart. Only meaningful with `write`. Its `GET /user` login is what the proxy treats as "own item" for comments. | +| `project.reference_repos[].write` | Unset (read-only). | Per-entry write opt-in, any of `issues`, `prs`. The **only** writes it unlocks on that repo: open an issue; open a PR / push a topic branch; comment on an issue or PR the hive itself opened. Never: merge, push to the default branch, labels or assignees on others' items, reviews, closing others' issues — the proxy refuses those deterministically (`reference_repo` in the violations log). The write is sent with a credential GitHub honours for *that* repo, never the hive's own installation token: the same GitHub App must already be **installed on the target org** (the hive discovers that installation and mints a token scoped to the one repo and the opted-in permissions). The hive owner cannot install it on a third-party org; the installation exists when that org runs its own hive. For a hand-written entry on an org without one, set `write_token_file`. Boot refuses (exit code 22) a hand-written write-mode entry with neither; a derived entry (below) is served read-only instead. Direct POSTs are used here, not the `hive-open-pr`/`hive-open-issue` relays, which only act on `project.repos`; a lane under `write_surface.enforce` therefore cannot write to a reference repo. | +| `project.reference_repos[].write_token_file` | Unset. | Path to a file holding a token (a maintainer PAT or fine-grained token) authorized for that repo; read on every use so a rotation needs no restart. Only meaningful with `write` on a hand-written entry; never consulted for a derived entry. Its `GET /user` login is what the proxy treats as "own item" for comments. | +| Derived `project.reference_repos` entries | Derived at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`). | Write-tier (`issues+prs`) entries the hive builds from an open request issue in an authorized repo that carries `triage/accepted` applied by a person with write access to that repo, when the repo's bound workflow model declares `intake.referenceRepoFields` ([#18139](https://github.com/hivecommons/hive/issues/18139)). A label applied by a bot or App account grants nothing. Hand-written entries win on the same repo; a repo in `project.repos` is never derived; the credential is the App installation on the requesting org, with a read-only fallback and one marked comment on the request issue when there is none. Closing the issue or removing the label drops the entry on the next pass. Never written to the overlay or runtime config: `Config.Save` strips them. Not editable in Settings (`409` on an edit). See [Reference repos derived from an accepted request](../agent-configuration.md#reference-repos-derived-from-an-accepted-request). | +| `intake.referenceRepoFields` (workflow model, `spec.intake`) | Unset on every model except `cncf-techdocs-assessment@curated-2026.11.2`. | Rendered issue-form headings (the `### <label>` lines in a request issue body, not field ids) whose values name repositories in scope. A model without this list derives no reference repos, so a hive bound to `curated-2026.11.1` must rebind to 11.2. Setting it changes the model's spec hash. See [workflow-models.md](../design/workflow-models.md#intake-hook-reference-repos-from-a-request-issue). | +| `intake.outOfScopeFields` (workflow model, `spec.intake`) | Unset. | Headings whose repositories are excluded even when also named under an in-scope heading. Only read when `referenceRepoFields` is set. | +| `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` | `10m`. Go duration; unset, unparseable or non-positive falls back to the default. | How often the hive re-lists accepted request issues between config reloads. Unchanged issues (same `updated_at`) cost no timeline or permission read; at most 50 labelled issues per repo per pass. | +| `referenceRepos[].derived_from`, `credential_state`, `credential_reason` (`/api/status` JSON) | Absent on hand-written entries. | `derived_from` is the request issue as `owner/repo#N`. `credential_state` is `write-tier` (installation found), `read-only` (no installation on the org; `credential_reason` names it) or `unresolved` (installation discovery failed on this pass; served read-only, nobody notified). These fields are `yaml:"-"` and never appear in a config file. | | `project.label_driven_approval_labels` | `["triage/accepted"]` | Labels whose trusted addition approves an issue on every label-driven repo without its own list. Ignored by repos without `label_driven: true`. Dashboard: Settings → Repos → "Default approval labels". | | `project.label_driven_trust_org_members` | `true` | Whether a public member of the repo's owning org who applies an approval label is trusted (Prow's own rule). Dashboard: Settings → Repos → "trust public org members". | | `project.label_driven_trusted_bots` | `["kubestellar-prow[bot]"]` | Bot logins whose label additions count as a trusted triager (Prow command bots relaying `/triage accepted`). `github-actions[bot]` is not in the default because it is the shared identity of every workflow in the repo, including ones triggered by untrusted events; operators whose Prow emulation runs as `github-actions[bot]` must add it explicitly per deployment. Dashboard: Settings → Repos → "Trusted bots". | diff --git a/src/docs/workflows/techdocs-assessment.md b/src/docs/workflows/techdocs-assessment.md index 236c1620bd..72d5ef94c9 100644 --- a/src/docs/workflows/techdocs-assessment.md +++ b/src/docs/workflows/techdocs-assessment.md @@ -158,6 +158,39 @@ rebuilt on every config load and never written to the overlay or runtime config, disappear when the issue closes or the label is removed. `curated-2026.11.1` is unchanged and declares no intake fields, so a hive that stays bound to it derives nothing. +#### Operator walkthrough: the Hive Commons pilot + +The first request through this path is +[cncf/techdocs#395](https://github.com/cncf/techdocs/issues/395), the Hive Commons assessment. +What the operator of the TechDocs hive (authorized repo `cncf/techdocs`) does and sees: + +1. **Rebind to 11.2.** Settings → Workflows → Marketplace, bind `cncf/techdocs` to + `cncf-techdocs-assessment@curated-2026.11.2`. This step is required: a hive still bound to + `curated-2026.11.1` derives nothing, however many requests are accepted. No + `reference_repos` entry is written by hand. +2. **A maintainer accepts the request.** Someone with write access to `cncf/techdocs` applies + `triage/accepted` to #395. A label applied by a bot grants nothing. +3. **Three derived entries appear** on the next pass (boot, config reload or every + `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL`, default `10m`): `hivecommons/docs`, + `hivecommons/hivecommons.github.io` and `hivecommons/hive`, each at the `issues+prs` tier with + `derived_from: cncf/techdocs#395`. The kick's `📚 REFERENCE REPOS` block lists them with + `(from cncf/techdocs#395)`. Settings → Reference repos shows them read-only with the source + link and `credential_state: write-tier`, because hivecommons has the App installed on every + repo. Had it not, the entries would read `read-only` with a reason, #395 would carry one + `hive:reference-repos-read-only` comment, and the hive would keep running. +4. **What the hive may do there.** Clone and read the three repos, read their issues and PRs, + open an issue, open a PR or push a topic branch, and comment on an issue or PR it opened + itself. The assessment deliverable still lives in `cncf/techdocs`. +5. **What it may not do.** Merge, push to a default branch, label or assign other people's + items, review, or close other people's issues. The proxy refuses each of these and logs + `reference_repo` in the violations log. `hivecommons` repos get no queue, no ACMM evaluation + and no work items. +6. **Closing #395 removes them.** When the request issue closes, or `triage/accepted` is + removed, the three entries, their proxy policies and their kick lines are gone on the next + pass. Nothing was ever written to the overlay or runtime config, so there is nothing to clean + up. A hand-written entry for one of the repos, if the operator adds one, takes precedence and + stays. + ## Not representable today These parts of the spec are documented in the model's annotations but not enforced by From 49cc0352cb641ec6cf6dfe26a6e2cbd54fbba535 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 03:06:55 +0000 Subject: [PATCH 207/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.253.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.253.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 12 ++++++++++++ .../added-18145-selfservice-registration-form.md | 1 - ...added-18158-derived-reference-repo-credentials.md | 1 - changelog.d/added-18160-techdocs-model-intake.md | 1 - ...urity-18166-selfservice-registration-hardening.md | 1 - 5 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/added-18145-selfservice-registration-form.md delete mode 100644 changelog.d/added-18158-derived-reference-repo-credentials.md delete mode 100644 changelog.d/added-18160-techdocs-model-intake.md delete mode 100644 changelog.d/security-18166-selfservice-registration-hardening.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c6303814..e2aa5a2bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.253.0) + +### Added + +- Hub dashboard: **Register your own hive** is now a form when the hub runs with `HIVE_HUB_SELFSERVICE_REGISTRATION` ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18145](https://github.com/hivecommons/hive/issues/18145)). Both CTAs (header and the "No hives yet" empty state) ask `GET /api/hub/selfservice/status` on load; with the flag off they keep linking to the self-host guide, with it on they open a modal that takes a name (shown with the fixed `byo-<login>-` prefix), org, one `owner/name` repository per line and an optional https dashboard URL, and posts to `/api/hub/selfservice/hives`. On success the heartbeat key is shown once with a copy button and a "copy before closing" warning, beside the hub URL and Docker Compose, Podman and Kubernetes snippets in tabs; the key lives only in the open dialog and is wiped from the DOM on close. Validation, not-an-admin (naming the refused repo), duplicate, per-user cap and GitHub-unreachable answers appear inline. A new **Self-hosted hives** section lists the signed-in user's own registrations with an online badge and last accepted heartbeat, **Rotate key** (confirm, then the new key once with the overlap note) and **Remove** (confirm, then revoke). +- A `project.reference_repos` entry derived from an accepted request issue resolves its credential from the GitHub App installation on the requesting org instead of refusing to boot: with an installation it keeps `write: [issues, prs]`, without one it is served read-only in memory and the request issue gets exactly one marked `hive:reference-repos-read-only` comment saying the hive can read but not open issues or PRs there (re-posted only when the read-only set changes, never on a plain reload). `write_token_file` is never consulted for a derived entry, a hand-written write-mode entry without a credential still exits 22, and `/api/status` `referenceRepos` entries now carry `derived_from`, `credential_state` (`write-tier`, `read-only`, `unresolved`) and `credential_reason` (#18139, #18158). +- The hub marketplace seeds a second curated TechDocs model, `cncf-techdocs-assessment@curated-2026.11.2`, identical to `curated-2026.11.1` except for a `spec.intake` block naming the assessment-request form headings ("Website or documentation repository", "Other repositories or sites in scope", "Out of scope") that yield derived write-tier reference repos once a maintainer accepts the request; `curated-2026.11.1` stays in the catalogue unchanged, so a hive still bound to it derives nothing until it rebinds (#18139, #18160). + +### Security + +- **Self-service hive registration hardening** ([#18166](https://github.com/hivecommons/hive/issues/18166), part of [#18060](https://github.com/hivecommons/hive/issues/18060)): `HIVE_HUB_SELFSERVICE_MAX_PER_USER` is now enforced again under the self-service store lock, so concurrent `POST /api/hub/selfservice/hives` requests from one user can no longer all pass the pre-ownership count and exceed the cap (the late refusal answers `429` and logs `selfservice_hive_refused` with `reason=per_user_cap`). New self-service IDs use `byo-<login>--<name>` instead of `byo-<login>-<name>`, so logins that contain hyphens can no longer collide or squat each other's IDs (`a-b` + `c` and `a` + `b-c` both used to become `byo-a-b-c`); a login containing `--` or starting or ending with `-` is refused with `reason=invalid_login`, and a name may not start with `-`. Hives registered before this change keep their existing IDs and keys. Repo references `.` and `..` (for example `owner/..`) are now refused everywhere repo refs are validated, so they can no longer be resolved as path segments in the GitHub ownership-check URL. + ## 2026-11-14 (v5.252.0) ### Added diff --git a/changelog.d/added-18145-selfservice-registration-form.md b/changelog.d/added-18145-selfservice-registration-form.md deleted file mode 100644 index cf8849ec37..0000000000 --- a/changelog.d/added-18145-selfservice-registration-form.md +++ /dev/null @@ -1 +0,0 @@ -- Hub dashboard: **Register your own hive** is now a form when the hub runs with `HIVE_HUB_SELFSERVICE_REGISTRATION` ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18145](https://github.com/hivecommons/hive/issues/18145)). Both CTAs (header and the "No hives yet" empty state) ask `GET /api/hub/selfservice/status` on load; with the flag off they keep linking to the self-host guide, with it on they open a modal that takes a name (shown with the fixed `byo-<login>-` prefix), org, one `owner/name` repository per line and an optional https dashboard URL, and posts to `/api/hub/selfservice/hives`. On success the heartbeat key is shown once with a copy button and a "copy before closing" warning, beside the hub URL and Docker Compose, Podman and Kubernetes snippets in tabs; the key lives only in the open dialog and is wiped from the DOM on close. Validation, not-an-admin (naming the refused repo), duplicate, per-user cap and GitHub-unreachable answers appear inline. A new **Self-hosted hives** section lists the signed-in user's own registrations with an online badge and last accepted heartbeat, **Rotate key** (confirm, then the new key once with the overlap note) and **Remove** (confirm, then revoke). diff --git a/changelog.d/added-18158-derived-reference-repo-credentials.md b/changelog.d/added-18158-derived-reference-repo-credentials.md deleted file mode 100644 index 7853a169cf..0000000000 --- a/changelog.d/added-18158-derived-reference-repo-credentials.md +++ /dev/null @@ -1 +0,0 @@ -- A `project.reference_repos` entry derived from an accepted request issue resolves its credential from the GitHub App installation on the requesting org instead of refusing to boot: with an installation it keeps `write: [issues, prs]`, without one it is served read-only in memory and the request issue gets exactly one marked `hive:reference-repos-read-only` comment saying the hive can read but not open issues or PRs there (re-posted only when the read-only set changes, never on a plain reload). `write_token_file` is never consulted for a derived entry, a hand-written write-mode entry without a credential still exits 22, and `/api/status` `referenceRepos` entries now carry `derived_from`, `credential_state` (`write-tier`, `read-only`, `unresolved`) and `credential_reason` (#18139, #18158). diff --git a/changelog.d/added-18160-techdocs-model-intake.md b/changelog.d/added-18160-techdocs-model-intake.md deleted file mode 100644 index b25e3492e7..0000000000 --- a/changelog.d/added-18160-techdocs-model-intake.md +++ /dev/null @@ -1 +0,0 @@ -- The hub marketplace seeds a second curated TechDocs model, `cncf-techdocs-assessment@curated-2026.11.2`, identical to `curated-2026.11.1` except for a `spec.intake` block naming the assessment-request form headings ("Website or documentation repository", "Other repositories or sites in scope", "Out of scope") that yield derived write-tier reference repos once a maintainer accepts the request; `curated-2026.11.1` stays in the catalogue unchanged, so a hive still bound to it derives nothing until it rebinds (#18139, #18160). diff --git a/changelog.d/security-18166-selfservice-registration-hardening.md b/changelog.d/security-18166-selfservice-registration-hardening.md deleted file mode 100644 index 2f2fc212c1..0000000000 --- a/changelog.d/security-18166-selfservice-registration-hardening.md +++ /dev/null @@ -1 +0,0 @@ -- **Self-service hive registration hardening** ([#18166](https://github.com/hivecommons/hive/issues/18166), part of [#18060](https://github.com/hivecommons/hive/issues/18060)): `HIVE_HUB_SELFSERVICE_MAX_PER_USER` is now enforced again under the self-service store lock, so concurrent `POST /api/hub/selfservice/hives` requests from one user can no longer all pass the pre-ownership count and exceed the cap (the late refusal answers `429` and logs `selfservice_hive_refused` with `reason=per_user_cap`). New self-service IDs use `byo-<login>--<name>` instead of `byo-<login>-<name>`, so logins that contain hyphens can no longer collide or squat each other's IDs (`a-b` + `c` and `a` + `b-c` both used to become `byo-a-b-c`); a login containing `--` or starting or ending with `-` is refused with `reason=invalid_login`, and a name may not start with `-`. Hives registered before this change keep their existing IDs and keys. Repo references `.` and `..` (for example `owner/..`) are now refused everywhere repo refs are validated, so they can no longer be resolved as path segments in the GitHub ownership-check URL. From 2a26d871c94dab72964bb73a47187f489fdf2743 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 22:21:57 -0500 Subject: [PATCH 208/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20reference=20re?= =?UTF-8?q?po=20cards=20show=20derived=20entries=20read-only=20with=20sour?= =?UTF-8?q?ce=20link=20and=20credential=20chip=20(#18161)=20(#18176)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/status referenceRepos gains derived_from_url, the request issue's web link built on the hive's GitHub host from the owner/repo#N provenance (empty when the reference is malformed). The repos page renders a derived reference card with a lock, a "from owner/repo#N" link and the credential chip (App installation / read-only with the reason / unresolved) and no edit or remove control. The Settings editor parts of #18161 (409 on editing a derived entry, filtering derived entries out of a save, editor rows) need the editor and PUT endpoint from PR #18154, which is not merged yet. Refs #18161 Hive-Run: hivecommons/hive#18161 Hive-Plan: #18139 step 5 Hive-Spec: derived-reference-repos#18139-5 Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...epos-card-shows-derived-reference-repos.md | 1 + src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 ++--- src/docs/api-reference/snapshots-and-style.md | 2 +- .../reference_repos_derived_18161_test.go | 96 +++++++++++++++++++ src/pkg/dashboard/server.go | 5 + src/pkg/dashboard/static-src/css/repos.css | 6 ++ .../dashboard/static-src/js/repos-render.js | 33 ++++++- src/pkg/dashboard/static/index.html | 39 +++++++- src/pkg/dashboard/status_builder.go | 23 ++++- 11 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 changelog.d/added-18161-repos-card-shows-derived-reference-repos.md create mode 100644 src/pkg/dashboard/reference_repos_derived_18161_test.go diff --git a/changelog.d/added-18161-repos-card-shows-derived-reference-repos.md b/changelog.d/added-18161-repos-card-shows-derived-reference-repos.md new file mode 100644 index 0000000000..fc7a623376 --- /dev/null +++ b/changelog.d/added-18161-repos-card-shows-derived-reference-repos.md @@ -0,0 +1 @@ +- The repos page's reference repo cards show an entry derived from an accepted request issue read-only: a lock, a `from owner/repo#N` link to the request issue and its credential chip (`🔑 App installation`, `read-only` with the reason, or `unresolved`), with no edit or remove control; `/api/status` `referenceRepos` entries now also carry `derived_from_url`, the request issue's link on the hive's GitHub host (#18139, #18161). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 0386da1826..a7090b4e48 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1734` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1735` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1736` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1739` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1740` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1741` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index e9ff3b72f0..3944e5c6b4 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1741` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1746` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 9f95dec44d..734a56a33f 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1714` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1715` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1716` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1722` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1727` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1728` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1729` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1730` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1731` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1732` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1733` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1719` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1720` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1721` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1727` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1732` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1733` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1734` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1735` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1736` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1737` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1738` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 658d675f9c..8b561b0f29 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1804` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1809` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/reference_repos_derived_18161_test.go b/src/pkg/dashboard/reference_repos_derived_18161_test.go new file mode 100644 index 0000000000..b540cb75d9 --- /dev/null +++ b/src/pkg/dashboard/reference_repos_derived_18161_test.go @@ -0,0 +1,96 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "os/exec" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// hivecommons/hive#18161: a derived reference repo carries a link to the +// request issue it came from, built on the hive's GitHub host, so the +// dashboard can show "from owner/repo#N" without guessing the host. +func TestStatusReferenceReposCarryDerivedFromURL(t *testing.T) { + srv := newFullServer(t) + cfg := srv.deps.Config + cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs", Write: []string{config.ReferenceWriteIssues}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier}, + {Repo: "hivecommons/hive"}, + } + srv.UpdateStatus(&StatusPayload{ReferenceRepos: buildReferenceRepos(cfg)}) + + rec := doGet(srv, "/api/status") + if rec.Code != http.StatusOK { + t.Fatalf("/api/status = %d %s", rec.Code, rec.Body.String()) + } + var body struct { + ReferenceRepos []map[string]any `json:"referenceRepos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode /api/status: %v", err) + } + got := map[string]map[string]any{} + for _, rr := range body.ReferenceRepos { + full, _ := rr["full"].(string) + got[full] = rr + } + if u := got["hivecommons/docs"]["derived_from_url"]; u != "https://github.com/cncf/techdocs/issues/395" { + t.Errorf("derived entry derived_from_url = %v, want the request issue URL", u) + } + if _, present := got["hivecommons/hive"]["derived_from_url"]; present { + t.Errorf("hand-written entry carries derived_from_url: %v", got["hivecommons/hive"]) + } +} + +func TestReferenceRepoSourceURL(t *testing.T) { + cases := []struct { + from, base, want string + }{ + {"cncf/techdocs#395", "https://github.com", "https://github.com/cncf/techdocs/issues/395"}, + {"cncf/techdocs#395", "https://ghe.example.com/", "https://ghe.example.com/cncf/techdocs/issues/395"}, + {"", "https://github.com", ""}, + {"cncf/techdocs", "https://github.com", ""}, + {"techdocs#395", "https://github.com", ""}, + {"cncf/techdocs#0", "https://github.com", ""}, + {"cncf/techdocs#x", "https://github.com", ""}, + {"a/b/c#1", "https://github.com", ""}, + } + for _, c := range cases { + if got := referenceRepoSourceURL(c.from, c.base); got != c.want { + t.Errorf("referenceRepoSourceURL(%q, %q) = %q, want %q", c.from, c.base, got, c.want) + } + } +} + +// hivecommons/hive#18161: the reference repo card renders a derived entry +// read-only with a lock, a "from owner/repo#N" link and its credential chip, +// and renders nothing extra for a hand-written entry. +func TestReferenceRepoCardRendersDerivedEntryReadOnly(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not on PATH — the derived reference repo card was NOT executed by this run") + } + html := indexHTML(t) + script := extractJSFunction(t, html, "esc") + "\n }\n" + + extractJSFunction(t, html, "renderReferenceRepoDerived") + "\n }\n" + ` +const assert = require('node:assert/strict'); +assert.equal(renderReferenceRepoDerived({full: 'hivecommons/hive', reference_tier: 'read-only'}), ''); +const w = renderReferenceRepoDerived({full: 'hivecommons/docs', derived_from: 'cncf/techdocs#395', derived_from_url: 'https://github.com/cncf/techdocs/issues/395', credential_state: 'write-tier'}); +assert.match(w, /🔒 from <a class="repo-reference-source" href="https:\/\/github.com\/cncf\/techdocs\/issues\/395"[^>]*>cncf\/techdocs#395<\/a>/); +assert.match(w, /data-credential-state="write-tier"[^>]*>🔑 App installation</); +const r = renderReferenceRepoDerived({full: 'acme/docs', derived_from: 'cncf/techdocs#395', credential_state: 'read-only', credential_reason: 'the hive\'s GitHub App is not installed on acme'}); +assert.match(r, /<span class="repo-reference-source">cncf\/techdocs#395<\/span>/); +assert.match(r, /data-credential-state="read-only" title="the hive's GitHub App is not installed on acme">read-only</); +for (const out of [w, r]) { + assert.doesNotMatch(out, /<button|<input|data-action=/, 'derived entries must offer no edit or delete control'); +} +console.log('ok'); +` + out, err := runNodeStdin(node, script) + if err != nil || !strings.Contains(string(out), "ok") { + t.Fatalf("node: %v\n%s", err, out) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 6e7bcd46da..f775e58388 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -1245,6 +1245,11 @@ type FrontendReferenceRepo struct { // DerivedFrom is the "owner/repo#N" request issue a derived entry came // from (hivecommons/hive#18139); empty for a hand-written entry. DerivedFrom string `json:"derived_from,omitempty"` + // DerivedFromURL is the web link to that request issue on the hive's + // GitHub host, so the dashboard links "from owner/repo#N" without + // re-deriving the host (hivecommons/hive#18161); empty when DerivedFrom + // is empty or not an "owner/repo#N" reference. + DerivedFromURL string `json:"derived_from_url,omitempty"` // CredentialState is a derived entry's resolved credential: // "write-tier", "read-only" or "unresolved" (hivecommons/hive#18158); // CredentialReason says why when it is not write-tier. diff --git a/src/pkg/dashboard/static-src/css/repos.css b/src/pkg/dashboard/static-src/css/repos.css index c8aa80659c..cfe7739018 100644 --- a/src/pkg/dashboard/static-src/css/repos.css +++ b/src/pkg/dashboard/static-src/css/repos.css @@ -79,6 +79,12 @@ .repo-reference-provider { color: var(--muted); } .repo-reference-link { color: inherit; text-decoration: none; } .repo-reference-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--indigo) 45%, transparent); background: color-mix(in srgb, var(--indigo) 12%, transparent); color: var(--indigo); cursor: help; } + .repo-reference-derived { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .repo-reference-source { color: inherit; font-weight: 600; } + .repo-reference-cred { font-weight: 700; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid currentColor; cursor: help; } + .repo-reference-cred-ok { color: var(--green); } + .repo-reference-cred-ro { color: var(--muted); } + .repo-reference-cred-warn { color: var(--yellow); } .repo-reference-note { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); } .repo-pause-btn { flex: 0 0 auto; margin-left: 0; font-size: 0.66rem; font-weight: 600; line-height: 1; padding: 3px 7px; border-radius: var(--r-pill); white-space: nowrap; cursor: pointer; border: 1px solid var(--border); background: transparent; color: var(--muted); opacity: 0.7; transition: opacity 0.15s, color 0.15s, border-color 0.15s; } .repo-automerge-toggle { flex: 0 0 auto; white-space: nowrap; gap: var(--sp-2); margin: 0; font-size: var(--fs-2xs); color: var(--muted); } diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index d9ad371d0e..0019592942 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -370,14 +370,43 @@ const tip = tier === 'read-only' ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; - return `<div class="repo-card repo-card-reference" data-repo="${esc(full)}"> + const derived = renderReferenceRepoDerived(rr); + return `<div class="repo-card repo-card-reference${derived ? ' repo-card-reference-derived' : ''}" data-repo="${esc(full)}"> <div class="repo-provider-label repo-reference-provider" title="This repo lives on ${esc(hostLabel)} and is outside this project">${esc(hostLabel)}</div> <div class="repo-name"><a class="repo-reference-link" href="${esc(base + '/' + full)}" target="_blank" rel="noopener" title="${esc(full)}">${esc(full)}</a><span class="repo-header-badges"><span class="repo-reference-pill" title="${esc(tip)}">📚 reference · ${esc(tier)}</span></span></div> - <div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> + ${derived}<div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> </div>`; }).join(''); } + /** + * A reference repo derived from an accepted request issue (#18139, #18161) + * is read-only here: a lock, a "from owner/repo#N" link to the request + * issue and its credential chip. It is rebuilt from the issue on every + * config load, so the dashboard never offers to edit or remove it; closing + * the issue or removing triage/accepted drops it. '' for hand-written. + */ + function renderReferenceRepoDerived(rr) { + const from = String((rr && rr.derived_from) || ''); + if (!from) return ''; + const url = String(rr.derived_from_url || ''); + const src = url + ? `<a class="repo-reference-source" href="${esc(url)}" target="_blank" rel="noopener" title="Open the request issue this entry is derived from">${esc(from)}</a>` + : `<span class="repo-reference-source">${esc(from)}</span>`; + const state = String(rr.credential_state || ''); + const reason = String(rr.credential_reason || ''); + const chips = { + 'write-tier': ['ok', '🔑 App installation', 'The hive\'s GitHub App is installed on this org; writes use a token minted for this repo only.'], + 'read-only': ['ro', 'read-only', reason || 'No App installation on this org, so the entry was downgraded to read-only.'], + 'unresolved': ['warn', 'unresolved', reason || 'Installation discovery failed; served read-only until the next reload resolves it.'], + }; + const chip = chips[state]; + const chipHTML = chip + ? ` <span class="repo-reference-cred repo-reference-cred-${chip[0]}" data-credential-state="${esc(state)}" title="${esc(chip[2])}">${esc(chip[1])}</span>` + : ''; + return `<div class="repo-reference-derived" title="Derived from an accepted request issue: read-only here, rebuilt on every config load. Close the issue or remove triage/accepted to drop it.">🔒 from ${src}${chipHTML}</div>`; + } + /** * Where a repo card points and what it calls its host (#14115). r.forge is * the repo's forge kind from /api/status; GitHub (and an older hive that diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index f728020095..0274c920bf 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -3329,6 +3329,12 @@ .repo-reference-provider { color: var(--muted); } .repo-reference-link { color: inherit; text-decoration: none; } .repo-reference-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--indigo) 45%, transparent); background: color-mix(in srgb, var(--indigo) 12%, transparent); color: var(--indigo); cursor: help; } + .repo-reference-derived { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .repo-reference-source { color: inherit; font-weight: 600; } + .repo-reference-cred { font-weight: 700; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid currentColor; cursor: help; } + .repo-reference-cred-ok { color: var(--green); } + .repo-reference-cred-ro { color: var(--muted); } + .repo-reference-cred-warn { color: var(--yellow); } .repo-reference-note { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); } .repo-pause-btn { flex: 0 0 auto; margin-left: 0; font-size: 0.66rem; font-weight: 600; line-height: 1; padding: 3px 7px; border-radius: var(--r-pill); white-space: nowrap; cursor: pointer; border: 1px solid var(--border); background: transparent; color: var(--muted); opacity: 0.7; transition: opacity 0.15s, color 0.15s, border-color 0.15s; } .repo-automerge-toggle { flex: 0 0 auto; white-space: nowrap; gap: var(--sp-2); margin: 0; font-size: var(--fs-2xs); color: var(--muted); } @@ -37413,14 +37419,43 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const tip = tier === 'read-only' ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; - return `<div class="repo-card repo-card-reference" data-repo="${esc(full)}"> + const derived = renderReferenceRepoDerived(rr); + return `<div class="repo-card repo-card-reference${derived ? ' repo-card-reference-derived' : ''}" data-repo="${esc(full)}"> <div class="repo-provider-label repo-reference-provider" title="This repo lives on ${esc(hostLabel)} and is outside this project">${esc(hostLabel)}</div> <div class="repo-name"><a class="repo-reference-link" href="${esc(base + '/' + full)}" target="_blank" rel="noopener" title="${esc(full)}">${esc(full)}</a><span class="repo-header-badges"><span class="repo-reference-pill" title="${esc(tip)}">📚 reference · ${esc(tier)}</span></span></div> - <div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> + ${derived}<div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> </div>`; }).join(''); } + /** + * A reference repo derived from an accepted request issue (#18139, #18161) + * is read-only here: a lock, a "from owner/repo#N" link to the request + * issue and its credential chip. It is rebuilt from the issue on every + * config load, so the dashboard never offers to edit or remove it; closing + * the issue or removing triage/accepted drops it. '' for hand-written. + */ + function renderReferenceRepoDerived(rr) { + const from = String((rr && rr.derived_from) || ''); + if (!from) return ''; + const url = String(rr.derived_from_url || ''); + const src = url + ? `<a class="repo-reference-source" href="${esc(url)}" target="_blank" rel="noopener" title="Open the request issue this entry is derived from">${esc(from)}</a>` + : `<span class="repo-reference-source">${esc(from)}</span>`; + const state = String(rr.credential_state || ''); + const reason = String(rr.credential_reason || ''); + const chips = { + 'write-tier': ['ok', '🔑 App installation', 'The hive\'s GitHub App is installed on this org; writes use a token minted for this repo only.'], + 'read-only': ['ro', 'read-only', reason || 'No App installation on this org, so the entry was downgraded to read-only.'], + 'unresolved': ['warn', 'unresolved', reason || 'Installation discovery failed; served read-only until the next reload resolves it.'], + }; + const chip = chips[state]; + const chipHTML = chip + ? ` <span class="repo-reference-cred repo-reference-cred-${chip[0]}" data-credential-state="${esc(state)}" title="${esc(chip[2])}">${esc(chip[1])}</span>` + : ''; + return `<div class="repo-reference-derived" title="Derived from an accepted request issue: read-only here, rebuilt on every config load. Close the issue or remove triage/accepted to drop it.">🔒 from ${src}${chipHTML}</div>`; + } + /** * Where a repo card points and what it calls its host (#14115). r.forge is * the repo's forge kind from /api/status; GitHub (and an older hive that diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 2a5186c7fd..c1694cd7c7 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -2388,12 +2388,33 @@ func buildReferenceRepos(cfg *config.Config) []FrontendReferenceRepo { tier := rr.WriteLabel() out = append(out, FrontendReferenceRepo{ Full: rr.Repo, Tier: tier, ReferenceTier: tier, - DerivedFrom: rr.DerivedFrom, CredentialState: rr.CredentialState, CredentialReason: rr.CredentialReason, + DerivedFrom: rr.DerivedFrom, DerivedFromURL: referenceRepoSourceURL(rr.DerivedFrom, cfg.GitHub.ResolvedBaseURL()), + CredentialState: rr.CredentialState, CredentialReason: rr.CredentialReason, }) } return out } +// referenceRepoSourceURL turns a derived entry's "owner/repo#N" provenance +// into the request issue's web URL on base (hivecommons/hive#18161). Anything +// that is not exactly owner/repo#N with a positive N yields "" so the +// dashboard shows the bare reference rather than a guessed link. +func referenceRepoSourceURL(derivedFrom, base string) string { + repo, num, ok := strings.Cut(strings.TrimSpace(derivedFrom), "#") + if !ok { + return "" + } + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" || strings.Contains(name, "/") { + return "" + } + n, err := strconv.Atoi(num) + if err != nil || n <= 0 { + return "" + } + return overviewItemURL("", repo, n, "issue", base) +} + func buildReposWithForge(cfg *config.Config, actionable *github.ActionableResult, govState governor.State, forge forgeOpenCounter) []FrontendRepo { repos := make([]FrontendRepo, 0, len(cfg.Project.Repos)) repoRows := make(map[string]string, len(cfg.Project.Repos)*2) From eb00a38cfe8ce4e1fb0e6bf82ea42f434e406939 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 22:28:52 -0500 Subject: [PATCH 209/223] =?UTF-8?q?=F0=9F=93=96=20docs:=20self-service=20h?= =?UTF-8?q?ive=20ID=20is=20byo-<login>--<name>=20(double=20hyphen)=20after?= =?UTF-8?q?=20#18170=20(#18179)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the /get-started self-host card, hub-deployment.md, the owner guide register-self-hosted-hive.md (including its example IDs) and the HIVE_MIRROR_HUB_HIVE_ID row in env-vars.md to the ID shape that the hardening in #18170 shipped: a double hyphen between login and name, since GitHub logins may contain a single hyphen. Refs #18166 #18147 Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18166-selfservice-docs-double-hyphen-id.md | 1 + src/docs/env-vars.md | 2 +- src/docs/hub-deployment.md | 2 +- src/docs/register-self-hosted-hive.md | 8 ++++---- src/pkg/hub/static/get-started.html | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md diff --git a/changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md b/changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md new file mode 100644 index 0000000000..5720139ec8 --- /dev/null +++ b/changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md @@ -0,0 +1 @@ +- The self-host card on `/get-started`, `hub-deployment.md`, `register-self-hosted-hive.md` and the `HIVE_MIRROR_HUB_HIVE_ID` row in `env-vars.md` now show the self-service hive ID as `byo-<login>--<name>`, the double-hyphen form that [#18166](https://github.com/hivecommons/hive/issues/18166) shipped, instead of the earlier single-hyphen form ([#18147](https://github.com/hivecommons/hive/issues/18147)). diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index f46c859cd0..11025415ba 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -72,7 +72,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_HUB_URL` | No | `hub.url` from config | Hub URL override for spoke heartbeats/registration. On the hub it is also the last environment variable consulted in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_MIRROR_HUB_URL` | No | unset (no mirror) | Base URL of a **second hub** that receives a report-only copy of every heartbeat the primary hub accepts ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18148](https://github.com/hivecommons/hive/issues/18148); `pkg/hub/spoke/heartbeat_mirror.go`). Must be an absolute `http(s)` URL and must differ from the primary hub. The copy leaves out the dashboard token hash, the App and webhook key fingerprints, authentication error text, the signed-in user list and the activity digest. The mirror's response is never applied: config, credential, upgrade, restart, breaker, access-list and visibility fields from it are counted and ignored, and logged at most once per hour as `mirror_hub_response_ignored` (field names only). A mirror failure never affects the primary heartbeat, liveness or the hive. Status appears in `GET /api/health/deep` under `hub_heartbeat.mirror` (host only). Read per beat; no restart needed. | | `HIVE_MIRROR_HUB_HEARTBEAT_KEY` | Required when `HIVE_MIRROR_HUB_URL` is set | none | Per-hive heartbeat key the **mirror** hub issued (its **Register your own hive** form, or an operator-minted key). Sent only to the mirror. The primary `HIVE_HEARTBEAT_KEY` is never offered to a mirror: with the URL set and this unset, nothing is sent and `hub_heartbeat.mirror.last_error` says so. Never logged. | -| `HIVE_MIRROR_HUB_HIVE_ID` | No | the primary `hive_id` | Overrides `hive_id` on the mirrored copy only, for a mirror hub that knows this hive under another id (for example its self-service `byo-<login>-<name>` id). | +| `HIVE_MIRROR_HUB_HIVE_ID` | No | the primary `hive_id` | Overrides `hive_id` on the mirrored copy only, for a mirror hub that knows this hive under another id (for example its self-service `byo-<login>--<name>` id). | | `HIVE_MIRROR_HUB_TIMEOUT` | No | `5s` | Timeout for one mirror POST, Go duration syntax. Deliberately shorter than the primary heartbeat budget; a slow mirror misses a beat rather than holding a goroutine. Invalid or non-positive values keep the default. | | `HIVE_SHARED_AGENT_POOL_ENABLED` | No | `false` | Spoke opt-in for Hub-coordinated CLI session sharing. Requires an in-process runtime and a Hub link; each Hive keeps its own workspace, UID, credentials, and tmux session. Idle CLI sessions are terminated after turns to return the global slot. | | `HIVE_SHARED_AGENT_POOL_MAX_SESSIONS` | No | `0` (disabled) | Hub-side maximum number of live CLI session leases. Set to `1` to serialize agent sessions across participating Hives. | diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index cf77bd3989..e7ce197017 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -110,7 +110,7 @@ All four are listed with the other hub variables in [env-vars.md](env-vars.md). #### What a registration does 1. The user is signed in through the hub's GitHub OAuth, so the hub holds their encrypted GitHub token. Without a stored token or a GitHub login the request is refused. -2. The user supplies a short name, an org, up to 20 `owner/name` repos and an `https` dashboard URL. The hub builds the ID as `byo-<github login>-<name>`. Nothing the user sends can produce an ID outside the `byo-` prefix, and a name that would borrow another namespace (such as `hosted-`) is refused. +2. The user supplies a short name, an org, up to 20 `owner/name` repos and an `https` dashboard URL. The hub builds the ID as `byo-<github login>--<name>`; the double hyphen keeps IDs unambiguous, since logins may contain `-` but never `--`, and a name may not start with `-`. Nothing the user sends can produce an ID outside the `byo-` prefix, and a name that would borrow another namespace (such as `hosted-`) is refused. 3. For every repo the hub calls GitHub with the user's own token and requires `permissions.admin`. `maintain` and `write` are not enough. One failing repo refuses the whole request with `403` naming that repo, and nothing is stored. 4. The hub checks the per-user cap and that the ID is not already taken, in the registry or by a hub-provisioned hive. 5. The hub mints a random heartbeat key, stores only its hash in `/data/hub-selfservice.json`, and returns the raw key in the `201` response together with Compose, Podman and Kubernetes snippets. The raw key appears nowhere else: not in the list route, not in a log line, not on disk. diff --git a/src/docs/register-self-hosted-hive.md b/src/docs/register-self-hosted-hive.md index afae153af7..a7020d9852 100644 --- a/src/docs/register-self-hosted-hive.md +++ b/src/docs/register-self-hosted-hive.md @@ -14,7 +14,7 @@ A self-hosted hive is complete on its own. Listing it on a hub is optional and g 1. Sign in to the hub with GitHub and open **My Hives**. 2. Click **Register your own hive**. -3. Fill in a short name, the GitHub org, the repos (as `owner/name`), and the `https` URL of your hive's dashboard. Your hive ID becomes `byo-<your login>-<name>`. +3. Fill in a short name, the GitHub org, the repos (as `owner/name`), and the `https` URL of your hive's dashboard. Your hive ID becomes `byo-<your login>--<name>`, with two hyphens between login and name. 4. Submit. The hub checks admin permission on each repo. One failing repo refuses the whole request and stores nothing. 5. **Copy the key now.** The dialog shows `HIVE_HEARTBEAT_KEY` exactly once and the hub keeps only a hash. Closing the dialog is final. If you miss it, rotate the key (below) rather than registering again. 6. Start the spoke with one of the snippets below, substituting the values from the dialog. @@ -32,7 +32,7 @@ services: image: ghcr.io/hivecommons/hive:stable environment: HIVE_HUB_URL: "https://hive.example.com" - HIVE_ID: "byo-yourlogin-yourname" + HIVE_ID: "byo-yourlogin--yourname" HIVE_HEARTBEAT_KEY: "<the key shown once>" ``` @@ -41,7 +41,7 @@ Podman: ```sh podman run -d --name hive \ -e HIVE_HUB_URL="https://hive.example.com" \ - -e HIVE_ID="byo-yourlogin-yourname" \ + -e HIVE_ID="byo-yourlogin--yourname" \ -e HIVE_HEARTBEAT_KEY="<the key shown once>" \ ghcr.io/hivecommons/hive:stable ``` @@ -56,7 +56,7 @@ metadata: type: Opaque stringData: HIVE_HUB_URL: "https://hive.example.com" - HIVE_ID: "byo-yourlogin-yourname" + HIVE_ID: "byo-yourlogin--yourname" HIVE_HEARTBEAT_KEY: "<the key shown once>" ``` diff --git a/src/pkg/hub/static/get-started.html b/src/pkg/hub/static/get-started.html index c2ef793c63..e6316a27b6 100644 --- a/src/pkg/hub/static/get-started.html +++ b/src/pkg/hub/static/get-started.html @@ -462,7 +462,7 @@ <h3>List your hive on this hub</h3> <li>Start the spoke. After its first accepted heartbeat it appears in Active Hives with the <strong>self-hosted</strong> type and in your <strong>Self-hosted hives</strong> list.</li> </ol> <p>Lost the key? Use <strong>Rotate key</strong> in Self-hosted hives; the old key keeps working for ten minutes so you can update a running spoke without a gap. <strong>Remove</strong> revokes the key and delists the hive at once. A hive that sends no accepted heartbeat for 24 hours is delisted automatically; register it again when it is back.</p> - <p>Your hive ID is <code>byo-<your login>-<name></code>. The hub lists it as directory metadata only: it never sends your hive credentials, upgrades or management actions, and you never receive <code>HIVE_HUB_SECRET</code>. The key you hold authenticates that one hive ID and nothing else.</p> + <p>Your hive ID is <code>byo-<your login>--<name></code> (two hyphens between login and name, since logins may contain a single hyphen). The hub lists it as directory metadata only: it never sends your hive credentials, upgrades or management actions, and you never receive <code>HIVE_HUB_SECRET</code>. The key you hold authenticates that one hive ID and nothing else.</p> <p style="margin-bottom:var(--sp-0)">A hub operator turns this on with <code>HIVE_HUB_SELFSERVICE_REGISTRATION</code>. On a hub where it is off, the Register your own hive button brings you here instead of opening the form; ask that hub's operator to enable it. If you would rather not run the infrastructure at all, use the hosted-hive wizard at the top of this page and the hub will provision and list one for you.</p> </div> </section> From 0358785a8ce88ced46c21a8b031cfeff380841cf Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:33:45 +0000 Subject: [PATCH 210/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20reference=20re?= =?UTF-8?q?pos=20editor=20(#18154)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dashboard): reference repos editor Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * fix: document /api/config/reference-repos in openapi.json Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 dashboard: roll back reference repos when config save fails PUT /api/config/reference-repos installed the new list on the proxy before persisting, and a save failure was only logged while returning 200. On failure, restore config and the proxy to the previous list and return 500 so enforced and persisted lists cannot diverge. Hive-Fix-Attempt: 1/6 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 dashboard: reference repos editor never stores derived entries; write_token_file stays operator-only Review follow-ups on the editor (#18136, #18154): - PUT drops derived entries (derived_from != "") from the incoming list before validating, applying and saving, and calls the new RemergeDerivedReferenceRepos dependency (wired to boot.remergeLastDerivedReferenceRepos) after the save, so the file holds only the operator's list and memory keeps the derived set, as a config reload does. - write_token_file is operator-only: the dashboard keeps a path the config file already holds for the same repo and refuses (400) to add or change one. The path names a file on the hive host that the credential check reads and sends to GitHub; a hosted hive's owner has no host access by design, so the editor must not pick it. The editor shows a file-set path as read-only text, carries derived_from through to the PUT body, and explains the rule. - Rollback after a failed save runs on a non-cancelling context; the 500 is documented in openapi.json. - Tests: handler (operator-only token, derived stripping and remerge with the saved file inspected), boot hook (derived policies kept, remerge), node test for the panel rendering, PUT body and refusal path. Signed-off-by: Andy Anderson <andy@clubanderson.com> * fix: tolerate text nodes in reference-repos editor token test Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Andy Anderson <andy@clubanderson.com> --- changelog.d/added-reference-repos-editor.md | 1 + dashboard/openapi.json | 95 ++++++ src/cmd/hive/main.go | 4 +- src/cmd/hive/reference_repos.go | 24 ++ src/cmd/hive/reference_repos_derive_test.go | 46 +++ src/docs/api-reference/configuration.md | 2 + src/docs/operator-reference/README.md | 4 +- src/pkg/config/reference_repos.go | 7 + src/pkg/dashboard/api.go | 3 + .../dashboard/api_config_reference_repos.go | 209 ++++++++++++ .../api_config_reference_repos_test.go | 298 ++++++++++++++++++ src/pkg/dashboard/audit.go | 1 + src/pkg/dashboard/deps.go | 14 + .../dashboard/reference_repos_editor.test.js | 150 +++++++++ .../dashboard/settings_parity_allowlist.txt | 3 - .../static-src/js/config-governor.js | 170 ++++++++++ src/pkg/dashboard/static/index.html | 170 ++++++++++ 17 files changed, 1195 insertions(+), 6 deletions(-) create mode 100644 changelog.d/added-reference-repos-editor.md create mode 100644 src/pkg/dashboard/api_config_reference_repos.go create mode 100644 src/pkg/dashboard/api_config_reference_repos_test.go create mode 100644 src/pkg/dashboard/reference_repos_editor.test.js diff --git a/changelog.d/added-reference-repos-editor.md b/changelog.d/added-reference-repos-editor.md new file mode 100644 index 0000000000..60fe6890e3 --- /dev/null +++ b/changelog.d/added-reference-repos-editor.md @@ -0,0 +1 @@ +- Dashboard: Settings → Repos has a Reference repos editor for `project.reference_repos`, so a hosted hive's owner can add, change and remove study-only repos without the config file; a save is refused with the loader's or boot's own message (including a write entry with no usable credential, or a `write_token_file` not already set in the file: that path stays operator-only) and otherwise applies to the GitHub proxy without a restart; derived entries are never written to the file (#18136). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 78c8ff9081..fcbff8a4bf 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -3595,6 +3595,101 @@ } } }, + "/api/config/reference-repos": { + "get": { + "tags": [ + "System" + ], + "summary": "Reference repos", + "description": "Returns project.reference_repos and the write vocabulary the editor offers. Owner only.", + "responses": { + "200": { + "description": "Current reference repos", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "reference_repos": { + "type": "array", + "items": { + "type": "object" + } + }, + "write_kinds": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "put": { + "tags": [ + "System" + ], + "summary": "Replace reference repos", + "description": "Replaces project.reference_repos after the same validation boot applies. Send an empty array to clear. Owner only.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "reference_repos" + ], + "properties": { + "reference_repos": { + "type": "array", + "items": { + "type": "object" + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Current reference repos", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "reference_repos": { + "type": "array", + "items": { + "type": "object" + } + }, + "write_kinds": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "Invalid list; nothing was stored" + }, + "500": { + "description": "The config file could not be written; the previous list is restored on the proxy and nothing changed" + } + } + } + }, "/api/config/stat-sources": { "get": { "tags": [ diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 32f602b168..175fb2da43 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1309,7 +1309,9 @@ func (b *boot) wireBootClosures() { ReInitFunc: func() { initAgentConfigDrivenSystems(b.cfg) }, - IoscanConfigApplied: b.reinstallIoscanClassifier, + IoscanConfigApplied: b.reinstallIoscanClassifier, + ApplyReferenceRepos: b.applyEditedReferenceRepos, + RemergeDerivedReferenceRepos: b.remergeLastDerivedReferenceRepos, ReviewConfigApplied: func(rc config.ReviewConfig) { if b.ghClient == nil { return diff --git a/src/cmd/hive/reference_repos.go b/src/cmd/hive/reference_repos.go index cec6d5f0c3..f6a645fbf8 100644 --- a/src/cmd/hive/reference_repos.go +++ b/src/cmd/hive/reference_repos.go @@ -204,3 +204,27 @@ func (b *boot) wireReferenceReposWith(rt *referenceRepoRuntime) { } b.startReferenceRepoDerivePoller(b.ctx, rt) } + +// applyEditedReferenceRepos is the dashboard editor's hook +// (hivecommons/hive#18136): it verifies the write credentials of the edited +// hand-written refs and, only when every entry passes, installs the rebuilt +// policies (derived entries from the last pass stay merged in). The error is +// returned to the owner verbatim and the previous list is kept. +func (b *boot) applyEditedReferenceRepos(ctx context.Context, refs []config.ReferenceRepo) error { + rt := b.referenceRepos + if rt == nil { + rt = b.newReferenceRepoRuntime() + } + if b.githubProxy == nil || b.cfg == nil { + return nil + } + rt.mu.Lock() + defer rt.mu.Unlock() + merged := mergedReferenceRepos(b.cfg, config.HandWrittenReferenceRepos(refs), rt.lastDerived, b.logger) + res, err := buildReferenceRepoPolicies(ctx, merged, rt.deps, b.logger) + if err != nil { + return err + } + b.githubProxy.SetReferenceRepos(res.policies) + return nil +} diff --git a/src/cmd/hive/reference_repos_derive_test.go b/src/cmd/hive/reference_repos_derive_test.go index 40582d12e0..06491b8221 100644 --- a/src/cmd/hive/reference_repos_derive_test.go +++ b/src/cmd/hive/reference_repos_derive_test.go @@ -420,3 +420,49 @@ func TestReferenceRepoDeriveInterval(t *testing.T) { } } } + +// The dashboard editor's hook (hivecommons/hive#18136) installs the edited +// hand-written list with the last derived set still merged in: a derived +// entry the editor echoes back is not re-verified as hand-written, a new +// hand-written entry gets its policy, and a removed one loses it. The +// remerge the handler runs after the save then restores the derived entries +// in memory under the new list. +func TestApplyEditedReferenceReposKeepsDerivedAndRemerges(t *testing.T) { + fake := newFakeRequestIssues() + b, rt := derivedTestBoot(t, fake, nil, config.ReferenceRepo{Repo: testHandWrittenRepo}) + b.wireReferenceReposWith(rt) + derivedBefore := len(rt.lastDerived) + if derivedBefore == 0 { + t.Fatal("precondition: boot derived entries") + } + + // Echo of the GET body (hand-written + derived) with the hand-written + // entry swapped for a new read-only one. + edited := append([]config.ReferenceRepo{{Repo: "other/ref"}}, rt.lastDerived...) + if err := b.applyEditedReferenceRepos(context.Background(), edited); err != nil { + t.Fatalf("applyEditedReferenceRepos: %v", err) + } + if _, ok := b.githubProxy.ReferenceRepoPolicyFor("other/ref"); !ok { + t.Error("new hand-written entry must get a proxy policy") + } + if _, ok := b.githubProxy.ReferenceRepoPolicyFor(testHandWrittenRepo); ok { + t.Error("removed hand-written entry must lose its proxy policy") + } + if pol, ok := b.githubProxy.ReferenceRepoPolicyFor(testDerivedWriteRepo); !ok || !pol.Issues || !pol.PRs { + t.Errorf("derived write-tier entry must stay enforced, got %+v ok=%v", pol, ok) + } + if len(rt.lastDerived) != derivedBefore { + t.Errorf("an edit must not change the derived set: %d -> %d", derivedBefore, len(rt.lastDerived)) + } + + // What the handler does after the file is written. + b.cfg.Project.ReferenceRepos = []config.ReferenceRepo{{Repo: "other/ref"}} + b.remergeLastDerivedReferenceRepos() + refs := b.cfg.Project.ReferenceRepos + if len(refs) != 1+derivedBefore || refs[0].Repo != "other/ref" || refs[0].Derived() { + t.Fatalf("derived entries must be merged back under the saved list, got %+v", refs) + } + if docs, ok := referenceRepoByName(refs, testDerivedWriteRepo); !ok || !docs.Derived() || docs.CredentialState != config.ReferenceCredentialWriteTier { + t.Fatalf("re-merged derived entry must keep its provenance and state, got %+v", docs) + } +} diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 2f68b18a3c..ef9a63eac6 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -101,6 +101,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:234` | | `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:541` | | `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:542` | +| `GET` | `/api/config/reference-repos` | Owner only | `project.reference_repos` ([#18136](https://github.com/hivecommons/hive/issues/18136)): `reference_repos` (`{repo, write, write_token_file}` plus `derived_from`, `credential_state`, `credential_reason` on a derived entry; always an array) and `write_kinds` | `pkg/dashboard/api.go:563` | +| `PUT` | `/api/config/reference-repos` | Owner only | Replace `project.reference_repos`; `{"reference_repos": []}` clears it. Refused with a 400 and nothing changed when the config loader would reject the list (bare name, repo also in `project.repos`, duplicate, unknown write kind, token file without write) or when a write entry has no credential boot would accept — the same message boot exits 22 with — or when an entry carries a `write_token_file` the config file does not already hold for that repo (the path is operator-only; a dashboard write opt-in uses the App installation). Entries with `derived_from` ([#18139](https://github.com/hivecommons/hive/issues/18139)) are never stored: they are dropped from the saved list and merged back in memory after the save. 500 when the file could not be written (proxy restored, nothing changed). On success the GitHub proxy's reference policies are replaced without a restart; audited as `config_reference_repos` | `pkg/dashboard/api.go:564` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:237` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:240` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:246` | diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index b4f7a7b99c..065cb1788a 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -251,9 +251,9 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `project.repo_policies[].label_driven` | Off (unset) for every repo. | Opt-in for repos whose maintainers accept and park issues with labels and their own bots (for example `needs-triage` → `triage/accepted`). On such a repo the un-park sweep posts no "What to reply" notice and never removes `needs-human`, `needs-decision` or `needs-direction` on a `/hive` comment alone; instead a trusted triager applying an approval label releases the issue and counts as human sign-off — see [Label-driven approval](#label-driven-approval). Editable from the dashboard: Settings → Repos → per-repo "Label-driven approvals (Prow-style)" (write or owner role). Details in [maintainer-commands.md](../maintainer-commands.md#label-driven-repositories). | | `project.repo_policies[].label_driven_approval_labels` | Unset: inherits `project.label_driven_approval_labels`. | Per-repo approval-label list; only read when `label_driven` is set. Editable from the dashboard sub-row of the repo's label-driven toggle. | | `project.repo_policies[].companion_repos` | Unset (no companions). | List of sibling repositories (`name` in the project org, or `org/name`) that this repo's tooling expects checked out beside it under the agent workspace root — for example a docs/website repo a spec tool registers at `<workspace>/<org>/<sibling>/…`. Contributor task prompts and resident-agent kicks carry a shallow-clone plan for each companion at `<workspace root>/<org>/<sibling>` (`$HIVE_WORKSPACE_DIR` for contributor agents, `$HOME/workspace` for resident agents), and agents are told that a tool reporting one of those paths missing means *clone it*, never park the issue. Self-references and malformed entries are dropped. See [running-at-level-6.md](../running-at-level-6.md#needless-parks). | -| `project.reference_repos` | Unset (none). | Repos **outside** the project the hive may *study* without managing them ([#18015](https://github.com/hivecommons/hive/issues/18015)) — the rare case of a hive whose deliverable is *about* another project, e.g. a CNCF TechDocs hive (authorized repo `cncf/techdocs`) assessing `hivecommons/docs`. Each entry is `owner/name` (bare names are rejected, as is any repo also in `project.repos`). Default tier is **read-only**: agents may `git clone`/fetch them unauthenticated over HTTPS and issue GitHub REST/GraphQL GETs; the proxy strips the hive's App token from every request to them and refuses every non-GET with a message naming `reference_repos`. The kick lists them under `📚 REFERENCE REPOS` so an authorized-repo item whose deliverables name one is treated as in scope (not out-of-scope/critical), and agents are told findings go in the deliverable inside the authorized repo, never as issues or PRs on the reference repo. The dashboard shows each as a minimal card with a `reference · <tier>` chip — no stats, no ACMM, no work items. Private reference repos are not readable (reads are unauthenticated). Example: <br>`project:`<br>` repos: [techdocs]`<br>` reference_repos:`<br>` - hivecommons/docs`<br>` - hivecommons/hivecommons.github.io`<br>` - {repo: hivecommons/hive, write: [issues, prs]}` | +| `project.reference_repos` | Unset (none). | Repos **outside** the project the hive may *study* without managing them ([#18015](https://github.com/hivecommons/hive/issues/18015)) — the rare case of a hive whose deliverable is *about* another project, e.g. a CNCF TechDocs hive (authorized repo `cncf/techdocs`) assessing `hivecommons/docs`. Each entry is `owner/name` (bare names are rejected, as is any repo also in `project.repos`). Default tier is **read-only**: agents may `git clone`/fetch them unauthenticated over HTTPS and issue GitHub REST/GraphQL GETs; the proxy strips the hive's App token from every request to them and refuses every non-GET with a message naming `reference_repos`. The kick lists them under `📚 REFERENCE REPOS` so an authorized-repo item whose deliverables name one is treated as in scope (not out-of-scope/critical), and agents are told findings go in the deliverable inside the authorized repo, never as issues or PRs on the reference repo. The dashboard shows each as a minimal card with a `reference · <tier>` chip — no stats, no ACMM, no work items. The owner can add, change and remove entries in Settings → Repos → Reference repos (`PUT /api/config/reference-repos`, [#18136](https://github.com/hivecommons/hive/issues/18136)); a save is refused with the loader's or boot's own message — including a write entry with no usable credential — and otherwise takes effect without a restart. Private reference repos are not readable (reads are unauthenticated). Example: <br>`project:`<br>` repos: [techdocs]`<br>` reference_repos:`<br>` - hivecommons/docs`<br>` - hivecommons/hivecommons.github.io`<br>` - {repo: hivecommons/hive, write: [issues, prs]}` | | `project.reference_repos[].write` | Unset (read-only). | Per-entry write opt-in, any of `issues`, `prs`. The **only** writes it unlocks on that repo: open an issue; open a PR / push a topic branch; comment on an issue or PR the hive itself opened. Never: merge, push to the default branch, labels or assignees on others' items, reviews, closing others' issues — the proxy refuses those deterministically (`reference_repo` in the violations log). The write is sent with a credential GitHub honours for *that* repo, never the hive's own installation token: the same GitHub App must already be **installed on the target org** (the hive discovers that installation and mints a token scoped to the one repo and the opted-in permissions). The hive owner cannot install it on a third-party org; the installation exists when that org runs its own hive. For a hand-written entry on an org without one, set `write_token_file`. Boot refuses (exit code 22) a hand-written write-mode entry with neither; a derived entry (below) is served read-only instead. Direct POSTs are used here, not the `hive-open-pr`/`hive-open-issue` relays, which only act on `project.repos`; a lane under `write_surface.enforce` therefore cannot write to a reference repo. | -| `project.reference_repos[].write_token_file` | Unset. | Path to a file holding a token (a maintainer PAT or fine-grained token) authorized for that repo; read on every use so a rotation needs no restart. Only meaningful with `write` on a hand-written entry; never consulted for a derived entry. Its `GET /user` login is what the proxy treats as "own item" for comments. | +| `project.reference_repos[].write_token_file` | Unset. | Path to a file holding a token (a maintainer PAT or fine-grained token) authorized for that repo; read on every use so a rotation needs no restart. Only meaningful with `write` on a hand-written entry; never consulted for a derived entry. Its `GET /user` login is what the proxy treats as "own item" for comments. File-only: the Settings editor shows a path the file set and keeps it across saves, but refuses to add or change one, since the path names a file on the hive host and a hosted hive's owner has no host access. | | Derived `project.reference_repos` entries | Derived at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`). | Write-tier (`issues+prs`) entries the hive builds from an open request issue in an authorized repo that carries `triage/accepted` applied by a person with write access to that repo, when the repo's bound workflow model declares `intake.referenceRepoFields` ([#18139](https://github.com/hivecommons/hive/issues/18139)). A label applied by a bot or App account grants nothing. Hand-written entries win on the same repo; a repo in `project.repos` is never derived; the credential is the App installation on the requesting org, with a read-only fallback and one marked comment on the request issue when there is none. Closing the issue or removing the label drops the entry on the next pass. Never written to the overlay or runtime config: `Config.Save` strips them. Not editable in Settings (`409` on an edit). See [Reference repos derived from an accepted request](../agent-configuration.md#reference-repos-derived-from-an-accepted-request). | | `intake.referenceRepoFields` (workflow model, `spec.intake`) | Unset on every model except `cncf-techdocs-assessment@curated-2026.11.2`. | Rendered issue-form headings (the `### <label>` lines in a request issue body, not field ids) whose values name repositories in scope. A model without this list derives no reference repos, so a hive bound to `curated-2026.11.1` must rebind to 11.2. Setting it changes the model's spec hash. See [workflow-models.md](../design/workflow-models.md#intake-hook-reference-repos-from-a-request-issue). | | `intake.outOfScopeFields` (workflow model, `spec.intake`) | Unset. | Headings whose repositories are excluded even when also named under an in-scope heading. Only read when `referenceRepoFields` is set. | diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go index 18e2b61d72..3cef388e5e 100644 --- a/src/pkg/config/reference_repos.go +++ b/src/pkg/config/reference_repos.go @@ -293,6 +293,13 @@ func (c *Config) ValidateReferenceRepos() error { return c.validateReferenceRepos() } +// ValidateReferenceRepos runs the config loader's project.reference_repos +// checks against p, so a dashboard edit is refused with exactly the message +// boot would print rather than leaving a hive.yaml that no longer loads. +func ValidateReferenceRepos(p ProjectConfig) error { + return (&Config{Project: p}).validateReferenceRepos() +} + // ReferenceRepoSlugs returns the reference repos as sorted "owner/name" // strings, for kicks and environment. func (c *Config) ReferenceRepoSlugs() []string { diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index f06ffa5784..0c57a32cd7 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -559,6 +559,9 @@ func (s *Server) RegisterAPI(deps *Dependencies) { s.registerAgentVariableRoutes() // Activity monitor row menu: Accept triage (api_repo_item_accept_triage.go). s.mux.HandleFunc("POST /api/repos/{owner}/{repo}/items/{number}/accept-triage", s.recordOperatorAction(s.handleRepoItemAcceptTriage)) + // project.reference_repos editor on the Repos tab (#18136, api_config_reference_repos.go). + s.mux.HandleFunc("GET /api/config/reference-repos", s.handleReferenceReposGet) + s.mux.HandleFunc("PUT /api/config/reference-repos", s.handleReferenceReposPut) } var ( diff --git a/src/pkg/dashboard/api_config_reference_repos.go b/src/pkg/dashboard/api_config_reference_repos.go new file mode 100644 index 0000000000..42345dedca --- /dev/null +++ b/src/pkg/dashboard/api_config_reference_repos.go @@ -0,0 +1,209 @@ +package dashboard + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// The project.reference_repos editor (hivecommons/hive#18136). +// +// Reference repos (#18015) are repositories the hive's agents may study but +// never manage. Until this editor they could only be set in the config file +// or overlay, which a hosted hive's owner cannot reach. Both endpoints are +// OWNER-ONLY: a write-mode entry lets agents open issues and PRs on a repo +// outside the project. +// +// A PUT is refused before anything changes unless the list passes the same +// checks boot applies — the loader's shape rules (config.ValidateReferenceRepos) +// and, when a GitHub proxy is wired, the write-credential check that would +// otherwise stop the next boot with exit 22 — so a dashboard edit can never +// leave the hive unbootable. + +// referenceReposSectionResponse is the GET/PUT response body. +type referenceReposSectionResponse struct { + // ReferenceRepos is the current list, in config order. Always an array + // (never null). + ReferenceRepos []config.ReferenceRepo `json:"reference_repos"` + // WriteKinds is the write vocabulary the editor offers. + WriteKinds []string `json:"write_kinds"` +} + +// handleReferenceReposGet returns project.reference_repos. +func (s *Server) handleReferenceReposGet(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "config unavailable", http.StatusServiceUnavailable) + return + } + jsonResponse(w, referenceReposSection(s.deps.Config)) +} + +// handleReferenceReposPut replaces project.reference_repos. The body must +// carry the whole list ({"reference_repos": [...]}); send an empty array to +// clear it. A missing key is refused rather than read as "clear". +func (s *Server) handleReferenceReposPut(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "config unavailable", http.StatusServiceUnavailable) + return + } + var body struct { + ReferenceRepos *[]config.ReferenceRepo `json:"reference_repos"` + } + if err := decodeBody(r, &body); err != nil { + jsonError(w, "invalid body", http.StatusBadRequest) + return + } + if body.ReferenceRepos == nil { + jsonError(w, `reference_repos is required; send {"reference_repos": []} to clear it`, http.StatusBadRequest) + return + } + cfg := s.deps.Config + + // --- validate before mutating anything --- + // Only the operator's entries are ever stored. A derived entry + // (hivecommons/hive#18139) lives in memory, is rebuilt from its request + // issue on every pass and must never land in hive.yaml, so the editor + // echoing one back is dropped here; the derived set is merged back in + // after the save. + prevRepos := cfg.Project.ReferenceRepos + next := normalizeReferenceRepos(config.HandWrittenReferenceRepos(*body.ReferenceRepos)) + if err := rejectDashboardTokenFiles(next, config.HandWrittenReferenceRepos(prevRepos)); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + candidate := cfg.Project + candidate.ReferenceRepos = next + if err := config.ValidateReferenceRepos(candidate); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if s.deps.ApplyReferenceRepos != nil { + if err := s.deps.ApplyReferenceRepos(r.Context(), next); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + } + + // --- apply --- + previous := len(config.HandWrittenReferenceRepos(prevRepos)) + cfg.Project.ReferenceRepos = next + if err := s.saveConfig(); err != nil { + // The proxy already enforces `next`; roll config and proxy back so + // the enforced list never diverges from the persisted one. The + // restore must finish even if the caller has gone away. + cfg.Project.ReferenceRepos = prevRepos + if s.deps.ApplyReferenceRepos != nil { + if rerr := s.deps.ApplyReferenceRepos(context.WithoutCancel(r.Context()), prevRepos); rerr != nil && s.logger != nil { + s.logger.Error("failed to restore proxy reference repos after persist failure", "error", rerr) + } + } + if s.logger != nil { + s.logger.Error("failed to persist config after reference repos update", "error", err) + } + jsonError(w, "failed to save config; reference repos unchanged", http.StatusInternalServerError) + return + } + // The file holds the hand-written list; memory gets the derived entries + // of the last pass back, as a config reload does. + if s.deps.RemergeDerivedReferenceRepos != nil { + s.deps.RemergeDerivedReferenceRepos() + } + writable := 0 + for _, rr := range next { + if !rr.ReadOnly() { + writable++ + } + } + s.auditFromRequest(r, "config_reference_repos", auditDetail( + "section", "reference_repos", + "repos", strconv.Itoa(len(next)), + "previous_repos", strconv.Itoa(previous), + "write_mode", strconv.Itoa(writable), + "list", strings.Join(cfg.ReferenceRepoSlugs(), "+"), + ), "") + if s.logger != nil { + s.logger.Info("reference repos updated via runtime settings", + "repos", len(next), "previous_repos", previous, "write_mode", writable) + } + jsonResponse(w, referenceReposSection(cfg)) +} + +// rejectDashboardTokenFiles refuses a write_token_file the dashboard would +// introduce or change. The path names a file on the hive host that the +// credential check reads and sends to GitHub as a bearer; a hosted hive's +// owner has no host access by design (hivecommons/hive#18154 review), so +// letting the editor pick the path would turn the owner role into an +// arbitrary-file read and a file-existence oracle over the hive process's +// filesystem. A path the operator already set in the config file for the +// same repo round-trips unchanged; everything else is "set it in the file". +func rejectDashboardTokenFiles(next, current []config.ReferenceRepo) error { + key := func(repo string) string { return strings.ToLower(strings.TrimSpace(repo)) } + fromFile := make(map[string]string, len(current)) + for _, rr := range current { + if rr.WriteTokenFile != "" { + fromFile[key(rr.Repo)] = rr.WriteTokenFile + } + } + for i, rr := range next { + if rr.WriteTokenFile == "" || fromFile[key(rr.Repo)] == rr.WriteTokenFile { + continue + } + return fmt.Errorf("project.reference_repos[%d]: write_token_file for %q can only be set in the config file by the hive operator; a write opt-in saved from the dashboard uses the GitHub App installation on the repo's org", i, rr.Repo) + } + return nil +} + +// normalizeReferenceRepos trims each entry and folds its write kinds to the +// canonical lower-case, de-duplicated "issues", "prs" order. Unknown kinds +// are kept as typed so the loader's validation names them. A token file +// without any write is kept too, for the same reason. +func normalizeReferenceRepos(in []config.ReferenceRepo) []config.ReferenceRepo { + out := make([]config.ReferenceRepo, 0, len(in)) + for _, rr := range in { + rr.Repo = strings.TrimSpace(rr.Repo) + rr.WriteTokenFile = strings.TrimSpace(rr.WriteTokenFile) + var known, unknown []string + seen := map[string]bool{} + for _, w := range rr.Write { + k := strings.ToLower(strings.TrimSpace(w)) + if k == "" || seen[k] { + continue + } + seen[k] = true + if k != config.ReferenceWriteIssues && k != config.ReferenceWritePRs { + unknown = append(unknown, strings.TrimSpace(w)) + } + } + if seen[config.ReferenceWriteIssues] { + known = append(known, config.ReferenceWriteIssues) + } + if seen[config.ReferenceWritePRs] { + known = append(known, config.ReferenceWritePRs) + } + rr.Write = append(known, unknown...) + if len(rr.Write) == 0 { + rr.Write = nil + } + out = append(out, rr) + } + return out +} + +// referenceReposSection renders project.reference_repos for the dashboard. +func referenceReposSection(cfg *config.Config) referenceReposSectionResponse { + refs := append([]config.ReferenceRepo{}, cfg.Project.ReferenceRepos...) + return referenceReposSectionResponse{ + ReferenceRepos: refs, + WriteKinds: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}, + } +} diff --git a/src/pkg/dashboard/api_config_reference_repos_test.go b/src/pkg/dashboard/api_config_reference_repos_test.go new file mode 100644 index 0000000000..e0d860ddc4 --- /dev/null +++ b/src/pkg/dashboard/api_config_reference_repos_test.go @@ -0,0 +1,298 @@ +package dashboard + +import ( + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// hivecommons/hive#18136: the owner-only project.reference_repos editor. + +const referenceReposPath = "/api/config/reference-repos" + +func decodeReferenceRepos(t *testing.T, body []byte) referenceReposSectionResponse { + t.Helper() + var got referenceReposSectionResponse + if err := json.Unmarshal(body, &got); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return got +} + +func TestReferenceReposGet_OwnerOnlyAndEmptyArray(t *testing.T) { + s := covApiServer(t) + if rec := doGetNoRole(s, referenceReposPath); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner GET: expected 403, got %d", rec.Code) + } + if rec := doPutNoRole(s, referenceReposPath, `{"reference_repos":[]}`); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner PUT: expected 403, got %d", rec.Code) + } + rec := doOwnerGet(s, referenceReposPath) + if rec.Code != http.StatusOK { + t.Fatalf("GET: expected 200, got %d", rec.Code) + } + if !strings.Contains(rec.Body.String(), `"reference_repos":[]`) { + t.Fatalf("default body must carry an empty array, not null: %s", rec.Body.String()) + } + got := decodeReferenceRepos(t, rec.Body.Bytes()) + if !reflect.DeepEqual(got.WriteKinds, []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}) { + t.Fatalf("write_kinds = %v", got.WriteKinds) + } +} + +func TestReferenceReposPut_RoundTripAndApplyHook(t *testing.T) { + s := covApiServer(t) + cfg := s.deps.Config + var applied []config.ReferenceRepo + s.deps.ApplyReferenceRepos = func(_ context.Context, refs []config.ReferenceRepo) error { + applied = refs + return nil + } + + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": []map[string]any{ + {"repo": " hivecommons/docs "}, + {"repo": "hivecommons/hive", "write": []string{"PRs", "issues", "prs"}}, + }}) + if rec.Code != http.StatusOK { + t.Fatalf("PUT: expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + want := []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + if got := decodeReferenceRepos(t, rec.Body.Bytes()).ReferenceRepos; !reflect.DeepEqual(got, want) { + t.Fatalf("PUT echoed %+v, want %+v", got, want) + } + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, want) { + t.Fatalf("config holds %+v, want %+v", cfg.Project.ReferenceRepos, want) + } + if !reflect.DeepEqual(applied, want) { + t.Fatalf("the proxy hook saw %+v, want %+v", applied, want) + } + // The kick and status read the live config: in force immediately. + if rr, ok := cfg.ReferenceRepoFor("HiveCommons/Hive"); !ok || !rr.WritesPRs() || !rr.WritesIssues() { + t.Fatalf("saved entry not visible to ReferenceRepoFor: %+v %v", rr, ok) + } + if back := decodeReferenceRepos(t, doOwnerGet(s, referenceReposPath).Body.Bytes()); !reflect.DeepEqual(back.ReferenceRepos, want) { + t.Fatalf("GET after PUT = %+v, want %+v", back.ReferenceRepos, want) + } + + // Clearing is explicit and reaches the proxy as an empty list. + if rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": []any{}}); rec.Code != http.StatusOK { + t.Fatalf("clear: expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + if len(cfg.Project.ReferenceRepos) != 0 || applied == nil || len(applied) != 0 { + t.Fatalf("clear left config=%+v applied=%+v", cfg.Project.ReferenceRepos, applied) + } +} + +func TestReferenceReposPut_RejectsWithLoaderMessage(t *testing.T) { + s := covApiServer(t) + cfg := s.deps.Config + // a/b carries an operator-set token file so the "token without write" + // case reaches the loader's message rather than the dashboard's + // operator-only refusal (tested separately). + prior := []config.ReferenceRepo{{Repo: "keep/me"}, {Repo: "a/b", Write: []string{config.ReferenceWriteIssues}, WriteTokenFile: "/t"}} + cfg.Project.ReferenceRepos = append([]config.ReferenceRepo{}, prior...) + + cases := []struct { + name string + entry config.ReferenceRepo + }{ + {"also authorized", config.ReferenceRepo{Repo: cfg.Project.Org + "/" + cfg.Project.Repos[0]}}, + {"bare name", config.ReferenceRepo{Repo: "docs"}}, + {"unknown write", config.ReferenceRepo{Repo: "a/b", Write: []string{"merge"}}}, + {"token without write", config.ReferenceRepo{Repo: "a/b", WriteTokenFile: "/t"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + candidate := cfg.Project + candidate.ReferenceRepos = []config.ReferenceRepo{tc.entry} + loaderErr := config.ValidateReferenceRepos(candidate) + if loaderErr == nil { + t.Fatalf("the loader accepts %+v; the case is wrong", tc.entry) + } + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": []config.ReferenceRepo{tc.entry}}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), jsonEscaped(t, loaderErr.Error())) { + t.Fatalf("response %s does not carry the loader's message %q", rec.Body.String(), loaderErr) + } + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, prior) { + t.Fatalf("a refused PUT changed the config: %+v", cfg.Project.ReferenceRepos) + } + }) + } + + if rec := doPut(s, referenceReposPath, map[string]any{}); rec.Code != http.StatusBadRequest { + t.Fatalf("missing key: expected 400, got %d", rec.Code) + } +} + +// A write-mode entry with no credential is refused with the message boot +// would exit 22 with, and nothing is stored, so the next boot still starts. +func TestReferenceReposPut_WriteWithoutCredentialRefused(t *testing.T) { + s := covApiServer(t) + cfg := s.deps.Config + // The production hook (cmd/hive applyReferenceRepos) runs the same + // verification boot runs; a hive with no App auth has no discover seam. + var bootMsg string + s.deps.ApplyReferenceRepos = func(ctx context.Context, refs []config.ReferenceRepo) error { + for _, rr := range refs { + if _, err := github.VerifyReferenceRepoWriteCredential(ctx, rr, "", nil, nil); err != nil { + bootMsg = err.Error() + return err + } + } + return nil + } + + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": []map[string]any{ + {"repo": "upstream/project", "write": []string{"issues"}}, + }}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", rec.Code, rec.Body.String()) + } + if bootMsg == "" || !strings.Contains(rec.Body.String(), jsonEscaped(t, bootMsg)) { + t.Fatalf("response %s does not carry the boot message %q", rec.Body.String(), bootMsg) + } + if !strings.Contains(bootMsg, github.ErrReferenceRepoNoCredential.Error()) { + t.Fatalf("boot message %q does not name the missing credential", bootMsg) + } + if len(cfg.Project.ReferenceRepos) != 0 { + t.Fatalf("a refused PUT stored %+v", cfg.Project.ReferenceRepos) + } +} + +// jsonEscaped returns s as it appears inside a JSON string. +func jsonEscaped(t *testing.T, s string) string { + t.Helper() + b, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + return strings.Trim(string(b), `"`) +} + +// write_token_file is operator-only (hivecommons/hive#18154 review): the +// path names a file on the hive host that the credential check reads and +// sends to GitHub, so the dashboard may keep a path the config file already +// holds for the same repo but never add or change one. +func TestReferenceReposPut_TokenFileIsOperatorOnly(t *testing.T) { + s := covApiServer(t) + cfg := s.deps.Config + prior := []config.ReferenceRepo{{Repo: "a/b", Write: []string{config.ReferenceWriteIssues}, WriteTokenFile: "/etc/hive/ab.token"}} + cfg.Project.ReferenceRepos = append([]config.ReferenceRepo{}, prior...) + + refused := []struct { + name string + body []map[string]any + }{ + {"new repo with a token file", []map[string]any{ + {"repo": "a/b", "write": []string{"issues"}, "write_token_file": "/etc/hive/ab.token"}, + {"repo": "c/d", "write": []string{"prs"}, "write_token_file": "/etc/passwd"}, + }}, + {"changed path on a file-set repo", []map[string]any{ + {"repo": "a/b", "write": []string{"issues"}, "write_token_file": "/etc/hive/other.token"}, + }}, + } + for _, tc := range refused { + t.Run(tc.name, func(t *testing.T) { + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": tc.body}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "write_token_file") || !strings.Contains(rec.Body.String(), "config file") { + t.Fatalf("refusal must name write_token_file and the config file: %s", rec.Body.String()) + } + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, prior) { + t.Fatalf("a refused PUT changed the config: %+v", cfg.Project.ReferenceRepos) + } + }) + } + + // The operator's path round-trips unchanged, next to a new App-backed entry. + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": []map[string]any{ + {"repo": "a/b", "write": []string{"issues"}, "write_token_file": "/etc/hive/ab.token"}, + {"repo": "c/d", "write": []string{"prs"}}, + }}) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + want := []config.ReferenceRepo{ + {Repo: "a/b", Write: []string{config.ReferenceWriteIssues}, WriteTokenFile: "/etc/hive/ab.token"}, + {Repo: "c/d", Write: []string{config.ReferenceWritePRs}}, + } + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, want) { + t.Fatalf("config holds %+v, want %+v", cfg.Project.ReferenceRepos, want) + } +} + +// A derived entry (hivecommons/hive#18139) echoed back by the editor is never +// stored: the hook and the file see only the hand-written list, and the +// derived set is merged back into memory after the save so /api/status and +// the kick keep it. +func TestReferenceReposPut_DerivedEntriesStrippedAndRemerged(t *testing.T) { + s := covApiServer(t) + cfg := s.deps.Config + cfg.SourcePath = filepath.Join(t.TempDir(), "hive.yaml") + derived := config.ReferenceRepo{Repo: "acme/docs", Write: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier} + cfg.Project.ReferenceRepos = []config.ReferenceRepo{{Repo: "keep/me"}, derived} + + var applied []config.ReferenceRepo + s.deps.ApplyReferenceRepos = func(_ context.Context, refs []config.ReferenceRepo) error { + applied = refs + return nil + } + remerged := 0 + s.deps.RemergeDerivedReferenceRepos = func() { + remerged++ + // What cmd/hive's remergeLastDerivedReferenceRepos does: the last + // derived set goes back under whatever hand-written list is in cfg. + cfg.Project.ReferenceRepos = append(config.HandWrittenReferenceRepos(cfg.Project.ReferenceRepos), derived) + } + + // The editor sends the list as GET returned it, plus one new entry. + got := decodeReferenceRepos(t, doOwnerGet(s, referenceReposPath).Body.Bytes()).ReferenceRepos + if len(got) != 2 || got[1].DerivedFrom != derived.DerivedFrom { + t.Fatalf("GET must show the derived entry with its provenance: %+v", got) + } + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": append(got, config.ReferenceRepo{Repo: "new/one"})}) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + handWritten := []config.ReferenceRepo{{Repo: "keep/me"}, {Repo: "new/one"}} + if !reflect.DeepEqual(applied, handWritten) { + t.Fatalf("the proxy hook must see only hand-written entries, got %+v", applied) + } + if remerged != 1 { + t.Fatalf("remerge hook called %d times, want 1", remerged) + } + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, append(handWritten, derived)) { + t.Fatalf("memory must hold the hand-written list with the derived entry merged back, got %+v", cfg.Project.ReferenceRepos) + } + if back := decodeReferenceRepos(t, rec.Body.Bytes()).ReferenceRepos; len(back) != 3 || back[2].DerivedFrom != derived.DerivedFrom { + t.Fatalf("PUT must echo the merged list, got %+v", back) + } + data, err := os.ReadFile(cfg.SourcePath) + if err != nil { + t.Fatal(err) + } + for _, leak := range []string{derived.Repo, derived.DerivedFrom, "derived_from", "credential_state"} { + if strings.Contains(string(data), leak) { + t.Fatalf("saved config leaks derived state %q:\n%s", leak, data) + } + } + if !strings.Contains(string(data), "new/one") || !strings.Contains(string(data), "keep/me") { + t.Fatalf("saved config must hold the hand-written entries:\n%s", data) + } +} diff --git a/src/pkg/dashboard/audit.go b/src/pkg/dashboard/audit.go index 6ab7db81e9..fe5f6e9309 100644 --- a/src/pkg/dashboard/audit.go +++ b/src/pkg/dashboard/audit.go @@ -105,6 +105,7 @@ var auditSensitiveActions = map[string]bool{ "config_governor_trajectory": true, "config_governor_watchdog": true, "config_governor_work_source": true, + "config_reference_repos": true, "config_review": true, "config_write_surface": true, "contributor-tier-changed": true, diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 898cf30714..cb22684935 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -203,6 +203,20 @@ type Dependencies struct { // whose min_priority the same PUT edits) can push the new values into // it. Nil means nothing caches. ReviewConfigApplied func(config.ReviewConfig) + // ApplyReferenceRepos is called by PUT /api/config/reference-repos with + // the validated list BEFORE it is stored: it verifies each write-mode + // entry's credential the way boot does and installs the result on the + // GitHub proxy, so the allow-list changes without a restart. A non-nil + // error refuses the edit with that message and nothing changes. Nil (no + // proxy, most tests) skips the check, matching boot, which only verifies + // credentials when a proxy is wired (hivecommons/hive#18136). + ApplyReferenceRepos func(ctx context.Context, refs []config.ReferenceRepo) error + // RemergeDerivedReferenceRepos is called after PUT /api/config/reference-repos + // has persisted the hand-written list: it puts the derived entries of the + // last pass (hivecommons/hive#18139) back into Config.Project.ReferenceRepos + // in memory, as the config reload hook does, so the kick and /api/status + // keep showing them. Nil (no deriver, most tests) leaves the list as saved. + RemergeDerivedReferenceRepos func() // RescanReposFunc re-enumerates every watched repository's open issues // and pull requests against the forge, enriches PR CI status, applies the // duplicate-PR claim guard and republishes the dashboard status snapshot diff --git a/src/pkg/dashboard/reference_repos_editor.test.js b/src/pkg/dashboard/reference_repos_editor.test.js new file mode 100644 index 0000000000..db75a30a42 --- /dev/null +++ b/src/pkg/dashboard/reference_repos_editor.test.js @@ -0,0 +1,150 @@ +// Settings → Repos → Reference repos editor (hivecommons/hive#18136): +// renderReferenceReposPanel paints one row per entry with write checkboxes +// and a remove button, shows an operator-set write_token_file as read-only +// text (never an input: the path is file-only), carries derived_from through +// to the PUT body so the server can drop derived entries, and shows the +// server's refusal verbatim without losing the edits. +// Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +const ASYNC_PREFIX = 'async '; +const fnSrc = name => { + let start = html.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + if (html.slice(start - ASYNC_PREFIX.length, start) === ASYNC_PREFIX) start -= ASYNC_PREFIX.length; + let depth = 0; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) return html.slice(start, i + 1); + } + throw new Error('unbalanced ' + name); +}; + +// Minimal fake DOM: elements with children, className, textContent (which +// clears children, as in the browser), attributes and listeners. +const makeDoc = () => { + const doc = {}; + const makeEl = tag => { + const el = { tag, children: [], listeners: {}, className: '', title: '', _text: '' }; + Object.defineProperty(el, 'textContent', { + get() { return el._text + el.children.map(c => c.textContent).join(''); }, + set(v) { el._text = String(v); el.children = []; }, + }); + el.appendChild = c => { el.children.push(c); return c; }; + el.addEventListener = (type, fn) => { (el.listeners[type] = el.listeners[type] || []).push(fn); }; + el.fire = (type, ev) => (el.listeners[type] || []).map(fn => fn(ev || {})); + el.find = pred => { + const out = []; + const walk = n => { if (pred(n)) out.push(n); (n.children || []).forEach(walk); }; + walk(el); + return out; + }; + return el; + }; + doc.createElement = makeEl; + doc.createTextNode = text => ({ tag: '#text', children: [], textContent: String(text) }); + doc.getElementById = () => null; + return doc; +}; + +const setup = ({ entries, kinds = ['issues', 'prs'], put }) => { + const doc = makeDoc(); + const toasts = []; + const calls = []; + const fetchStub = async (url, opts) => { + calls.push({ url, opts, body: JSON.parse(opts.body) }); + return put(calls[calls.length - 1]); + }; + const render = new Function('document', 'fetch', 'showToast', '_inceptionAuthHeaders', 'saveErrorMessage', + fnSrc('referenceRepoNode') + '\n' + fnSrc('referenceRepoYamlHint') + '\n' + fnSrc('renderReferenceReposPanel') + '\nreturn renderReferenceReposPanel;')( + doc, fetchStub, (m, k) => toasts.push([m, k]), () => ({}), async res => res.errorMessage); + const host = doc.createElement('div'); + render(host, entries, kinds); + return { doc, host, toasts, calls }; +}; + +const rows = host => host.find(n => n.className === 'config-list-item config-repo-row'); +const inputs = (node, type) => node.find(n => n.tag === 'input' && n.type === type); +const buttons = (node, text) => node.find(n => n.tag === 'button' && n.textContent === text); + +test('rows carry the repo, one checkbox per write kind, the tier and a remove button; no token input anywhere', () => { + const { host } = setup({ entries: [{ repo: 'hivecommons/docs' }, { repo: 'acme/tools', write: ['issues'] }], put: () => { throw new Error('no save'); } }); + const r = rows(host); + assert.strictEqual(r.length, 2); + assert.strictEqual(r[0].find(n => n.className === 'config-repo-name')[0].textContent, 'hivecommons/docs'); + assert.strictEqual(inputs(r[0], 'checkbox').length, 2); + assert.strictEqual(r[0].find(n => n.className === 'rt-inherit')[0].textContent, 'read-only'); + assert.strictEqual(r[1].find(n => n.className === 'rt-inherit')[0].textContent, 'issues'); + assert.deepStrictEqual(inputs(r[1], 'checkbox').map(b => b.checked), [true, false]); + assert.strictEqual(inputs(host, 'text').length, 1, 'the only text input is the owner/repo add box'); + assert.strictEqual(buttons(r[0], '✕').length, 1); + assert.ok(host.find(n => n.className === 'settings-yaml-hint').length >= 3, 'yaml hints for repo, write[] and write_token_file'); +}); + +test('an operator-set write_token_file is shown as read-only text and sent back unchanged', async () => { + const entries = [{ repo: 'acme/tools', write: ['issues'], write_token_file: '/etc/hive/acme.token' }]; + const { host, calls, toasts } = setup({ + entries, + put: call => ({ ok: true, json: async () => ({ reference_repos: call.body.reference_repos, write_kinds: ['issues', 'prs'] }) }), + }); + const token = host.find(n => (n.className || '').includes('cfg-reference-repo-token')); + assert.strictEqual(token.length, 1); + assert.strictEqual(token[0].tag, 'span', 'the path is text, not an editable input'); + assert.strictEqual(token[0].textContent, 'token file: /etc/hive/acme.token'); + assert.match(token[0].title, /config file/); + await buttons(host, 'Save reference repos')[0].fire('click')[0]; + assert.strictEqual(calls.length, 1); + assert.strictEqual(calls[0].opts.method, 'PUT'); + assert.deepStrictEqual(calls[0].body, { reference_repos: [{ repo: 'acme/tools', write: ['issues'], write_token_file: '/etc/hive/acme.token' }] }); + assert.deepStrictEqual(toasts, [['Reference repos saved', 'success']]); +}); + +test('a derived entry keeps derived_from in the PUT body; a hand-written one has none', async () => { + const entries = [ + { repo: 'hivecommons/docs' }, + { repo: 'acme/docs', write: ['issues', 'prs'], derived_from: 'cncf/techdocs#395', credential_state: 'write-tier' }, + ]; + const { host, calls } = setup({ + entries, + put: call => ({ ok: true, json: async () => ({ reference_repos: call.body.reference_repos }) }), + }); + await buttons(host, 'Save reference repos')[0].fire('click')[0]; + assert.deepStrictEqual(calls[0].body.reference_repos, [ + { repo: 'hivecommons/docs' }, + { repo: 'acme/docs', write: ['issues', 'prs'], derived_from: 'cncf/techdocs#395' }, + ]); +}); + +test('add, toggle and remove edit the list; a refused save shows the server message and keeps the edits', async () => { + const message = 'project.reference_repos[0]: "myorg/repo1" is already in project.repos'; + const { host, calls, toasts } = setup({ + entries: [{ repo: 'old/one' }], + put: () => ({ ok: false, status: 400, errorMessage: message }), + }); + const add = inputs(host, 'text')[0]; + add.value = 'https://github.com/myorg/repo1.git'; + buttons(host, '+ Add')[0].fire('click'); + let r = rows(host); + assert.deepStrictEqual(r.map(x => x.find(n => n.className === 'config-repo-name')[0].textContent), ['old/one', 'myorg/repo1']); + const prs = inputs(r[1], 'checkbox')[1]; + prs.checked = true; + prs.fire('change'); + r = rows(host); + assert.strictEqual(r[1].find(n => n.className === 'rt-inherit')[0].textContent, 'prs'); + buttons(r[0], '✕')[0].fire('click'); + r = rows(host); + assert.strictEqual(r.length, 1); + + await buttons(host, 'Save reference repos')[0].fire('click')[0]; + assert.deepStrictEqual(calls[0].body, { reference_repos: [{ repo: 'myorg/repo1', write: ['prs'] }] }); + const status = host.find(n => n.className === 'fs-sm text-danger'); + assert.strictEqual(status.length, 1); + assert.strictEqual(status[0].textContent, message); + assert.strictEqual(toasts.length, 0); + assert.strictEqual(rows(host).length, 1, 'edits survive a refused save'); + assert.strictEqual(buttons(host, 'Save reference repos')[0].disabled, false); +}); diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 9568f96b5d..620b15a13c 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -485,9 +485,6 @@ project.paused_repos[].by # TODO #11854 project.paused_repos[].reason # TODO #11854 project.paused_repos[].repo # TODO #11854 project.primary_repo # TODO #11854 -project.reference_repos[].repo # file-only by design (#18015): rare study-only repos, set in the overlay -project.reference_repos[].write[] # file-only by design (#18015) -project.reference_repos[].write_token_file # file-only by design (#18015) project.repo_policies[].acmm_last_automatic.at # TODO #11854 project.repo_policies[].acmm_last_automatic.direction # TODO #11854 project.repo_policies[].acmm_last_automatic.evidence_ids[] # TODO #11854 diff --git a/src/pkg/dashboard/static-src/js/config-governor.js b/src/pkg/dashboard/static-src/js/config-governor.js index 85e0a0216d..0ee2364798 100644 --- a/src/pkg/dashboard/static-src/js/config-governor.js +++ b/src/pkg/dashboard/static-src/js/config-governor.js @@ -998,6 +998,9 @@ const KICK_LIST_CAP_PR_DEFAULT = 50; function renderGovRepos(repos) { + /* Reference repos (#18136): painted asynchronously from their own + owner-only endpoint, like the Security tab's write allowlist. */ + loadReferenceRepos(); const primary = _configState.data.primaryRepo || ''; const selfAuthDefault = _configState.data.selfAuthorizationHold !== false; const selfAuthEnvLocked = !!_configState.data.selfAuthorizationHoldEnvLocked; @@ -1144,6 +1147,8 @@ <div id="cfg-repo-add-status" style="font-size:0.72rem;margin-top:var(--sp-3)"></div> ${defaultWarn} </div> + <div class="settings-subsection-title mt-4">Reference repos ${yamlHint('project.reference_repos[]')} ${settingHelpMark('settings', 'repos', 'Reference repos', 'Repositories outside this project that agents may clone and read but do not manage: no queue, no kicks, no work items. Read-only by default; a write opt-in allows opening issues and/or PRs there with a credential the hive verifies on save. A repo is either monitored above or a reference repo, never both.')}</div> + <div id="cfg-reference-repos-host"><div class="fs-sm text-muted">Loading…</div></div> ${anyLabelDriven ? `<div class="ld-defaults" id="cfg-label-driven-defaults"> <div class="config-toggle"> <div class="config-toggle-switch ${ldApproval.trustOrgMembers !== false ? 'on' : ''} ${ldLocked ? 'disabled' : ''}" data-action="${ldLocked ? '' : 'toggleLabelDrivenTrustOrgMembers'}" data-arg-types="t" title="Prow's own rule: public members of the repo's org may apply the approval label. Off: only triage/write/maintain/admin collaborators and trusted bots."></div> @@ -1186,6 +1191,171 @@ </div>`; } + /* Reference repos editor (#18136): project.reference_repos, loaded from + and saved to the owner-only /api/config/reference-repos endpoint. + Rendered with DOM nodes and textContent only, so a repo name or token + path from hive.yaml is never parsed as markup. The server applies the + config loader's checks and the boot-time write-credential check, and + its error message is shown as-is; nothing is saved on an error. */ + async function loadReferenceRepos() { + let host = null; + try { + const r = await fetch('/api/config/reference-repos'); + host = document.getElementById('cfg-reference-repos-host'); + if (!host) return; + if (r.status === 403) { + host.textContent = ''; + host.appendChild(referenceRepoNode('div', 'fs-sm text-muted', 'Only the hive owner can view and edit reference repos.')); + return; + } + if (!r.ok) throw new Error('HTTP ' + r.status); + const d = await r.json(); + renderReferenceReposPanel(host, d.reference_repos || [], d.write_kinds || ['issues', 'prs']); + } catch (e) { + host = host || document.getElementById('cfg-reference-repos-host'); + if (!host) return; + host.textContent = ''; + host.appendChild(referenceRepoNode('div', 'fs-sm text-danger', 'Failed to load reference repos: ' + e.message)); + } + } + + /* referenceRepoNode builds one element whose content is plain text. */ + function referenceRepoNode(tag, className, text) { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; + } + + /* referenceRepoYamlHint is yamlHint() as a DOM node. */ + function referenceRepoYamlHint(path) { + const hint = referenceRepoNode('span', 'settings-yaml-hint', 'YAML '); + hint.appendChild(referenceRepoNode('code', '', path)); + return hint; + } + + /* renderReferenceReposPanel paints the editable list and wires Save. + entries is edited in place until Save; a failed Save keeps the edits. */ + function renderReferenceReposPanel(host, entries, kinds) { + if (!host) return; + /* derived_from is carried through so the server can tell a derived + entry (rebuilt from its request issue, never saved) from an + operator's. */ + entries = entries.map(e => ({ repo: e.repo || '', write: (e.write || []).slice(), write_token_file: e.write_token_file || '', derived_from: e.derived_from || '' })); + host.textContent = ''; + host.appendChild(referenceRepoNode('p', 'config-help', + 'Read-only by default: agents clone and read these repos and are never told they are out of scope, but every write is refused. ' + + 'Tick issues and/or prs to let agents open issues or PRs there; that needs the GitHub App installed on the repo\'s org. ' + + 'A write_token_file is a path on the hive host and can only be set in the config file by the operator; one that is set is shown here and kept. ' + + 'Saving checks the credential and applies the change without a restart.')); + const list = referenceRepoNode('div', 'config-list'); + host.appendChild(list); + const statusEl = referenceRepoNode('div', 'fs-sm'); + const setStatus = (msg, cls) => { statusEl.textContent = msg; statusEl.className = 'fs-sm ' + (cls || 'text-muted'); }; + + const paintRows = () => { + list.textContent = ''; + if (!entries.length) list.appendChild(referenceRepoNode('div', 'fs-sm text-muted mb-3', 'No reference repos configured')); + entries.forEach((entry, i) => { + const row = referenceRepoNode('div', 'config-list-item config-repo-row'); + row.appendChild(referenceRepoNode('span', 'config-repo-name', entry.repo)); + kinds.forEach(kind => { + const label = referenceRepoNode('label', 'rt-check'); + const box = document.createElement('input'); + box.type = 'checkbox'; + box.checked = entry.write.includes(kind); + box.addEventListener('change', () => { + entry.write = kinds.filter(k => k === kind ? box.checked : entry.write.includes(k)); + paintRows(); + }); + label.appendChild(box); + label.appendChild(document.createTextNode(' write: ' + kind)); + row.appendChild(label); + }); + /* The token file is operator-only (set in the config file, never + from here): shown read-only when the file set one, so a save + round-trips it instead of dropping it. */ + if (entry.write_token_file) { + const token = referenceRepoNode('span', 'text-muted fs-sm cfg-reference-repo-token', 'token file: ' + entry.write_token_file); + token.title = 'write_token_file set in the config file for ' + entry.repo + '. Only the hive operator can change it.'; + row.appendChild(token); + } + row.appendChild(referenceRepoNode('span', 'rt-inherit', entry.write.length ? entry.write.join('+') : 'read-only')); + const remove = referenceRepoNode('button', 'remove-item', '\u2715'); + remove.title = 'Remove ' + entry.repo + ' from the reference repos'; + remove.addEventListener('click', () => { entries.splice(i, 1); paintRows(); }); + row.appendChild(remove); + list.appendChild(row); + }); + }; + + const add = referenceRepoNode('div', 'config-list-add'); + const input = document.createElement('input'); + input.type = 'text'; + input.id = 'cfg-reference-repo-input'; + input.placeholder = 'owner/repo'; + input.title = 'An explicit owner/repo outside this project. It must not also be a monitored repo.'; + const addBtn = referenceRepoNode('button', 'hv-btn btn-primary', '+ Add'); + const doAdd = () => { + const repo = input.value.trim().replace(/^https?:\/\/[^/]+\//, '').replace(/\.git$/, '').replace(/\/+$/, ''); + if (!repo) return; + if (entries.some(e => e.repo.toLowerCase() === repo.toLowerCase())) { + setStatus(repo + ' is already listed', 'text-danger'); + return; + } + entries.push({ repo, write: [], write_token_file: '' }); + input.value = ''; + setStatus(''); + paintRows(); + }; + addBtn.addEventListener('click', doAdd); + input.addEventListener('keydown', e => { if (e.key === 'Enter') { e.preventDefault(); doAdd(); } }); + add.appendChild(input); + add.appendChild(addBtn); + host.appendChild(add); + + const hints = referenceRepoNode('div', 'config-help'); + hints.appendChild(referenceRepoYamlHint('project.reference_repos[].repo')); + hints.appendChild(document.createTextNode(' ')); + hints.appendChild(referenceRepoYamlHint('project.reference_repos[].write[]')); + hints.appendChild(document.createTextNode(' ')); + hints.appendChild(referenceRepoYamlHint('project.reference_repos[].write_token_file')); + host.appendChild(hints); + + const actions = referenceRepoNode('div', 'config-list-add'); + const saveBtn = referenceRepoNode('button', 'hv-btn btn-primary btn-sm', 'Save reference repos'); + actions.appendChild(saveBtn); + actions.appendChild(statusEl); + host.appendChild(actions); + paintRows(); + + saveBtn.addEventListener('click', async () => { + saveBtn.disabled = true; + setStatus('Saving…'); + try { + const body = entries.map(e => { + const out = { repo: e.repo }; + if (e.write.length) out.write = e.write; + if (e.write.length && e.write_token_file) out.write_token_file = e.write_token_file; + if (e.derived_from) out.derived_from = e.derived_from; + return out; + }); + const res = await fetch('/api/config/reference-repos', { + method: 'PUT', + headers: _inceptionAuthHeaders(), + body: JSON.stringify({ reference_repos: body }) + }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const d = await res.json(); + renderReferenceReposPanel(host, d.reference_repos || [], d.write_kinds || kinds); + showToast('Reference repos saved', 'success'); + } catch (e) { + saveBtn.disabled = false; + setStatus(e.message, 'text-danger'); + } + }); + } + function setDefaultRepo(repo) { _configState.data.primaryRepo = repo; markDirty('repos', 'primaryRepo', repo); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 0274c920bf..48f2850d8c 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -64282,6 +64282,9 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> const KICK_LIST_CAP_PR_DEFAULT = 50; function renderGovRepos(repos) { + /* Reference repos (#18136): painted asynchronously from their own + owner-only endpoint, like the Security tab's write allowlist. */ + loadReferenceRepos(); const primary = _configState.data.primaryRepo || ''; const selfAuthDefault = _configState.data.selfAuthorizationHold !== false; const selfAuthEnvLocked = !!_configState.data.selfAuthorizationHoldEnvLocked; @@ -64428,6 +64431,8 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> <div id="cfg-repo-add-status" style="font-size:0.72rem;margin-top:var(--sp-3)"></div> ${defaultWarn} </div> + <div class="settings-subsection-title mt-4">Reference repos ${yamlHint('project.reference_repos[]')} ${settingHelpMark('settings', 'repos', 'Reference repos', 'Repositories outside this project that agents may clone and read but do not manage: no queue, no kicks, no work items. Read-only by default; a write opt-in allows opening issues and/or PRs there with a credential the hive verifies on save. A repo is either monitored above or a reference repo, never both.')}</div> + <div id="cfg-reference-repos-host"><div class="fs-sm text-muted">Loading…</div></div> ${anyLabelDriven ? `<div class="ld-defaults" id="cfg-label-driven-defaults"> <div class="config-toggle"> <div class="config-toggle-switch ${ldApproval.trustOrgMembers !== false ? 'on' : ''} ${ldLocked ? 'disabled' : ''}" data-action="${ldLocked ? '' : 'toggleLabelDrivenTrustOrgMembers'}" data-arg-types="t" title="Prow's own rule: public members of the repo's org may apply the approval label. Off: only triage/write/maintain/admin collaborators and trusted bots."></div> @@ -64470,6 +64475,171 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> </div>`; } + /* Reference repos editor (#18136): project.reference_repos, loaded from + and saved to the owner-only /api/config/reference-repos endpoint. + Rendered with DOM nodes and textContent only, so a repo name or token + path from hive.yaml is never parsed as markup. The server applies the + config loader's checks and the boot-time write-credential check, and + its error message is shown as-is; nothing is saved on an error. */ + async function loadReferenceRepos() { + let host = null; + try { + const r = await fetch('/api/config/reference-repos'); + host = document.getElementById('cfg-reference-repos-host'); + if (!host) return; + if (r.status === 403) { + host.textContent = ''; + host.appendChild(referenceRepoNode('div', 'fs-sm text-muted', 'Only the hive owner can view and edit reference repos.')); + return; + } + if (!r.ok) throw new Error('HTTP ' + r.status); + const d = await r.json(); + renderReferenceReposPanel(host, d.reference_repos || [], d.write_kinds || ['issues', 'prs']); + } catch (e) { + host = host || document.getElementById('cfg-reference-repos-host'); + if (!host) return; + host.textContent = ''; + host.appendChild(referenceRepoNode('div', 'fs-sm text-danger', 'Failed to load reference repos: ' + e.message)); + } + } + + /* referenceRepoNode builds one element whose content is plain text. */ + function referenceRepoNode(tag, className, text) { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; + } + + /* referenceRepoYamlHint is yamlHint() as a DOM node. */ + function referenceRepoYamlHint(path) { + const hint = referenceRepoNode('span', 'settings-yaml-hint', 'YAML '); + hint.appendChild(referenceRepoNode('code', '', path)); + return hint; + } + + /* renderReferenceReposPanel paints the editable list and wires Save. + entries is edited in place until Save; a failed Save keeps the edits. */ + function renderReferenceReposPanel(host, entries, kinds) { + if (!host) return; + /* derived_from is carried through so the server can tell a derived + entry (rebuilt from its request issue, never saved) from an + operator's. */ + entries = entries.map(e => ({ repo: e.repo || '', write: (e.write || []).slice(), write_token_file: e.write_token_file || '', derived_from: e.derived_from || '' })); + host.textContent = ''; + host.appendChild(referenceRepoNode('p', 'config-help', + 'Read-only by default: agents clone and read these repos and are never told they are out of scope, but every write is refused. ' + + 'Tick issues and/or prs to let agents open issues or PRs there; that needs the GitHub App installed on the repo\'s org. ' + + 'A write_token_file is a path on the hive host and can only be set in the config file by the operator; one that is set is shown here and kept. ' + + 'Saving checks the credential and applies the change without a restart.')); + const list = referenceRepoNode('div', 'config-list'); + host.appendChild(list); + const statusEl = referenceRepoNode('div', 'fs-sm'); + const setStatus = (msg, cls) => { statusEl.textContent = msg; statusEl.className = 'fs-sm ' + (cls || 'text-muted'); }; + + const paintRows = () => { + list.textContent = ''; + if (!entries.length) list.appendChild(referenceRepoNode('div', 'fs-sm text-muted mb-3', 'No reference repos configured')); + entries.forEach((entry, i) => { + const row = referenceRepoNode('div', 'config-list-item config-repo-row'); + row.appendChild(referenceRepoNode('span', 'config-repo-name', entry.repo)); + kinds.forEach(kind => { + const label = referenceRepoNode('label', 'rt-check'); + const box = document.createElement('input'); + box.type = 'checkbox'; + box.checked = entry.write.includes(kind); + box.addEventListener('change', () => { + entry.write = kinds.filter(k => k === kind ? box.checked : entry.write.includes(k)); + paintRows(); + }); + label.appendChild(box); + label.appendChild(document.createTextNode(' write: ' + kind)); + row.appendChild(label); + }); + /* The token file is operator-only (set in the config file, never + from here): shown read-only when the file set one, so a save + round-trips it instead of dropping it. */ + if (entry.write_token_file) { + const token = referenceRepoNode('span', 'text-muted fs-sm cfg-reference-repo-token', 'token file: ' + entry.write_token_file); + token.title = 'write_token_file set in the config file for ' + entry.repo + '. Only the hive operator can change it.'; + row.appendChild(token); + } + row.appendChild(referenceRepoNode('span', 'rt-inherit', entry.write.length ? entry.write.join('+') : 'read-only')); + const remove = referenceRepoNode('button', 'remove-item', '\u2715'); + remove.title = 'Remove ' + entry.repo + ' from the reference repos'; + remove.addEventListener('click', () => { entries.splice(i, 1); paintRows(); }); + row.appendChild(remove); + list.appendChild(row); + }); + }; + + const add = referenceRepoNode('div', 'config-list-add'); + const input = document.createElement('input'); + input.type = 'text'; + input.id = 'cfg-reference-repo-input'; + input.placeholder = 'owner/repo'; + input.title = 'An explicit owner/repo outside this project. It must not also be a monitored repo.'; + const addBtn = referenceRepoNode('button', 'hv-btn btn-primary', '+ Add'); + const doAdd = () => { + const repo = input.value.trim().replace(/^https?:\/\/[^/]+\//, '').replace(/\.git$/, '').replace(/\/+$/, ''); + if (!repo) return; + if (entries.some(e => e.repo.toLowerCase() === repo.toLowerCase())) { + setStatus(repo + ' is already listed', 'text-danger'); + return; + } + entries.push({ repo, write: [], write_token_file: '' }); + input.value = ''; + setStatus(''); + paintRows(); + }; + addBtn.addEventListener('click', doAdd); + input.addEventListener('keydown', e => { if (e.key === 'Enter') { e.preventDefault(); doAdd(); } }); + add.appendChild(input); + add.appendChild(addBtn); + host.appendChild(add); + + const hints = referenceRepoNode('div', 'config-help'); + hints.appendChild(referenceRepoYamlHint('project.reference_repos[].repo')); + hints.appendChild(document.createTextNode(' ')); + hints.appendChild(referenceRepoYamlHint('project.reference_repos[].write[]')); + hints.appendChild(document.createTextNode(' ')); + hints.appendChild(referenceRepoYamlHint('project.reference_repos[].write_token_file')); + host.appendChild(hints); + + const actions = referenceRepoNode('div', 'config-list-add'); + const saveBtn = referenceRepoNode('button', 'hv-btn btn-primary btn-sm', 'Save reference repos'); + actions.appendChild(saveBtn); + actions.appendChild(statusEl); + host.appendChild(actions); + paintRows(); + + saveBtn.addEventListener('click', async () => { + saveBtn.disabled = true; + setStatus('Saving…'); + try { + const body = entries.map(e => { + const out = { repo: e.repo }; + if (e.write.length) out.write = e.write; + if (e.write.length && e.write_token_file) out.write_token_file = e.write_token_file; + if (e.derived_from) out.derived_from = e.derived_from; + return out; + }); + const res = await fetch('/api/config/reference-repos', { + method: 'PUT', + headers: _inceptionAuthHeaders(), + body: JSON.stringify({ reference_repos: body }) + }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const d = await res.json(); + renderReferenceReposPanel(host, d.reference_repos || [], d.write_kinds || kinds); + showToast('Reference repos saved', 'success'); + } catch (e) { + saveBtn.disabled = false; + setStatus(e.message, 'text-danger'); + } + }); + } + function setDefaultRepo(repo) { _configState.data.primaryRepo = repo; markDirty('repos', 'primaryRepo', repo); From 974e1a4b35d4cdc4c6592b13095600407731916a Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 03:40:01 +0000 Subject: [PATCH 211/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.254.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.254.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 12 ++++++++++++ .../added-18139-derived-reference-repos-feature.md | 1 - changelog.d/added-18148-mirror-hub.md | 1 - .../added-18159-derive-reference-repos-on-load.md | 1 - .../changed-18147-selfservice-registration-docs.md | 1 - 5 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/added-18139-derived-reference-repos-feature.md delete mode 100644 changelog.d/added-18148-mirror-hub.md delete mode 100644 changelog.d/added-18159-derive-reference-repos-on-load.md delete mode 100644 changelog.d/changed-18147-selfservice-registration-docs.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e2aa5a2bb3..82ff6f33a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.254.0) + +### Added + +- A hive now derives write-tier `project.reference_repos` entries from an accepted request issue instead of needing the operator to hand-write them ([#18139](https://github.com/hivecommons/hive/issues/18139)). When an authorized repo's bound workflow model declares `intake.referenceRepoFields` (first: `cncf-techdocs-assessment@curated-2026.11.2`; hives bound to 11.1 must rebind), every repository the request form names in scope becomes an `issues+prs` reference repo once a person with write access applies `triage/accepted`; a bot applying the label grants nothing. The credential is the App installation on the requesting org, with a read-only fallback and one marked comment on the request issue when there is none, so a derived entry never fails boot. Entries are rebuilt at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`), vanish when the issue closes or loses the label, are never written to any saved config, yield to a hand-written entry on the same repo, carry `(from owner/repo#N)` in the kick, and appear read-only in Settings with their source and credential state. Docs: `agent-configuration.md`, `operator-reference/README.md`, `design/workflow-models.md`, `workflows/techdocs-assessment.md` (#18157, #18158, #18159, #18160, #18161, #18162). +- Spoke: optional **report-only heartbeat mirror** to a second hub ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18148](https://github.com/hivecommons/hive/issues/18148)). With `HIVE_MIRROR_HUB_URL` and `HIVE_MIRROR_HUB_HEARTBEAT_KEY` set (the key comes from registering the hive on that hub, for example through its **Register your own hive** form), every heartbeat the primary hub accepts is also POSTed, in its own goroutine with its own short timeout (`HIVE_MIRROR_HUB_TIMEOUT`, default 5s), to the mirror hub as a reduced copy: the dashboard token hash, the App and webhook key fingerprints, authentication error text, the signed-in user list and the activity digest are left out, and `HIVE_MIRROR_HUB_HIVE_ID` can override the reported `hive_id`. The mirror's response is never applied: upgrade, restart, branch switch, breaker, GitHub App config, funded gateway, project claim, access list, visibility and cluster id from the mirror are counted and ignored, and logged at most once per hour as `mirror_hub_response_ignored` (field names only). A mirror failure never affects the primary heartbeat, liveness or the hive. The primary key is never sent to the mirror; without a mirror key nothing is sent. `GET /api/health/deep` shows the mirror under `hub_heartbeat.mirror` (`url_host`, `last_ok`, `last_attempt`, `last_error`, `ignored_fields`, `last_ignored_fields`), host only, never the key. +- `project.reference_repos` entries are now derived from accepted request issues at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`), in memory only: for each authorized repo whose bound workflow model declares `intake.referenceRepoFields`, the hive lists open issues carrying `triage/accepted`, reads who applied the label from the timeline and their repository permission, runs the request-form parser and installs the result on the proxy at the `issues+prs` tier (hand-written entries win on the same repo, authorized repos are never derived, duplicates keep the first). Closing the request issue or removing the label drops the entry, its proxy policy and its kick line on the next pass; a saved overlay or runtime config never contains a derived entry; the kick's REFERENCE REPOS block appends `(from owner/repo#N)` and, for a read-only derived entry, the reason (#18139, #18159). + +### Changed + +- The hub's `/get-started#self-host` card now describes self-service registration (sign in, Register your own hive, copy the one-time key, paste the Compose, Podman or Kubernetes snippet) instead of telling self-hosters to open an issue and obtain `HIVE_HUB_SECRET` from the operator. `hub-deployment.md` gains a Self-service registration section for operators, `env-vars.md` notes that `byo-` hives hold only `HIVE_HEARTBEAT_KEY`, and a new `register-self-hosted-hive.md` walks owners through the flow ([#18147](https://github.com/hivecommons/hive/issues/18147), part of [#18060](https://github.com/hivecommons/hive/issues/18060)). + ## 2026-11-14 (v5.253.0) ### Added diff --git a/changelog.d/added-18139-derived-reference-repos-feature.md b/changelog.d/added-18139-derived-reference-repos-feature.md deleted file mode 100644 index 45083826d0..0000000000 --- a/changelog.d/added-18139-derived-reference-repos-feature.md +++ /dev/null @@ -1 +0,0 @@ -- A hive now derives write-tier `project.reference_repos` entries from an accepted request issue instead of needing the operator to hand-write them ([#18139](https://github.com/hivecommons/hive/issues/18139)). When an authorized repo's bound workflow model declares `intake.referenceRepoFields` (first: `cncf-techdocs-assessment@curated-2026.11.2`; hives bound to 11.1 must rebind), every repository the request form names in scope becomes an `issues+prs` reference repo once a person with write access applies `triage/accepted`; a bot applying the label grants nothing. The credential is the App installation on the requesting org, with a read-only fallback and one marked comment on the request issue when there is none, so a derived entry never fails boot. Entries are rebuilt at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`), vanish when the issue closes or loses the label, are never written to any saved config, yield to a hand-written entry on the same repo, carry `(from owner/repo#N)` in the kick, and appear read-only in Settings with their source and credential state. Docs: `agent-configuration.md`, `operator-reference/README.md`, `design/workflow-models.md`, `workflows/techdocs-assessment.md` (#18157, #18158, #18159, #18160, #18161, #18162). diff --git a/changelog.d/added-18148-mirror-hub.md b/changelog.d/added-18148-mirror-hub.md deleted file mode 100644 index 4adf5488b1..0000000000 --- a/changelog.d/added-18148-mirror-hub.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke: optional **report-only heartbeat mirror** to a second hub ([#18060](https://github.com/hivecommons/hive/issues/18060), [#18148](https://github.com/hivecommons/hive/issues/18148)). With `HIVE_MIRROR_HUB_URL` and `HIVE_MIRROR_HUB_HEARTBEAT_KEY` set (the key comes from registering the hive on that hub, for example through its **Register your own hive** form), every heartbeat the primary hub accepts is also POSTed, in its own goroutine with its own short timeout (`HIVE_MIRROR_HUB_TIMEOUT`, default 5s), to the mirror hub as a reduced copy: the dashboard token hash, the App and webhook key fingerprints, authentication error text, the signed-in user list and the activity digest are left out, and `HIVE_MIRROR_HUB_HIVE_ID` can override the reported `hive_id`. The mirror's response is never applied: upgrade, restart, branch switch, breaker, GitHub App config, funded gateway, project claim, access list, visibility and cluster id from the mirror are counted and ignored, and logged at most once per hour as `mirror_hub_response_ignored` (field names only). A mirror failure never affects the primary heartbeat, liveness or the hive. The primary key is never sent to the mirror; without a mirror key nothing is sent. `GET /api/health/deep` shows the mirror under `hub_heartbeat.mirror` (`url_host`, `last_ok`, `last_attempt`, `last_error`, `ignored_fields`, `last_ignored_fields`), host only, never the key. diff --git a/changelog.d/added-18159-derive-reference-repos-on-load.md b/changelog.d/added-18159-derive-reference-repos-on-load.md deleted file mode 100644 index 6f1ad327b3..0000000000 --- a/changelog.d/added-18159-derive-reference-repos-on-load.md +++ /dev/null @@ -1 +0,0 @@ -- `project.reference_repos` entries are now derived from accepted request issues at boot, on every config reload and every `HIVE_REFERENCE_REPOS_DERIVE_INTERVAL` (default `10m`), in memory only: for each authorized repo whose bound workflow model declares `intake.referenceRepoFields`, the hive lists open issues carrying `triage/accepted`, reads who applied the label from the timeline and their repository permission, runs the request-form parser and installs the result on the proxy at the `issues+prs` tier (hand-written entries win on the same repo, authorized repos are never derived, duplicates keep the first). Closing the request issue or removing the label drops the entry, its proxy policy and its kick line on the next pass; a saved overlay or runtime config never contains a derived entry; the kick's REFERENCE REPOS block appends `(from owner/repo#N)` and, for a read-only derived entry, the reason (#18139, #18159). diff --git a/changelog.d/changed-18147-selfservice-registration-docs.md b/changelog.d/changed-18147-selfservice-registration-docs.md deleted file mode 100644 index 6e86ff7232..0000000000 --- a/changelog.d/changed-18147-selfservice-registration-docs.md +++ /dev/null @@ -1 +0,0 @@ -- The hub's `/get-started#self-host` card now describes self-service registration (sign in, Register your own hive, copy the one-time key, paste the Compose, Podman or Kubernetes snippet) instead of telling self-hosters to open an issue and obtain `HIVE_HUB_SECRET` from the operator. `hub-deployment.md` gains a Self-service registration section for operators, `env-vars.md` notes that `byo-` hives hold only `HIVE_HEARTBEAT_KEY`, and a new `register-self-hosted-hive.md` walks owners through the flow ([#18147](https://github.com/hivecommons/hive/issues/18147), part of [#18060](https://github.com/hivecommons/hive/issues/18060)). From 6cfca8aa242d62306bd9de1283b7f9749cd7fc26 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 22:46:44 -0500 Subject: [PATCH 212/223] =?UTF-8?q?=E2=9C=A8=20dashboard:=20reference=20re?= =?UTF-8?q?pos=20editor=20shows=20derived=20entries=20read-only=20and=20re?= =?UTF-8?q?fuses=20to=20edit=20them=20(#18139=20step=205)=20(#18182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Derived entries (derived_from != "", #18174) are the request issue's grant. The Settings editor now renders them as read-only rows with a lock, a "from owner/repo#N" link to the issue on the hive's GitHub host, a credential-state chip (write-tier / read-only / unresolved) and the reason inline and on hover; no write checkboxes, no remove button. Hand-written rows are unchanged. PUT /api/config/reference-repos compares the incoming list with the live derived set and answers 409, naming the issue to change instead, when a derived entry is edited, left out, sent with another source, not derived on this hive, or replaced by a hand-written entry for the same repo. Nothing is stored on a 409. The editor toasts the 409 and keeps the edits. Tests: handler 409 cases plus a hand-written edit around an echoed derived entry; node tests for the derived row and the 409 toast. openapi.json and the API reference document the 409; changelog fragment added; static/index.html regenerated. Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...-18161-derived-reference-repos-settings.md | 1 + dashboard/openapi.json | 3 + src/docs/api-reference/configuration.md | 2 +- .../dashboard/api_config_reference_repos.go | 56 ++++++++++++++++ .../api_config_reference_repos_test.go | 53 +++++++++++++++ .../dashboard/reference_repos_editor.test.js | 65 +++++++++++++++++-- .../static-src/css/config-dialog.css | 7 ++ .../static-src/js/config-governor.js | 44 ++++++++++++- src/pkg/dashboard/static/index.html | 51 ++++++++++++++- 9 files changed, 271 insertions(+), 11 deletions(-) create mode 100644 changelog.d/added-18161-derived-reference-repos-settings.md diff --git a/changelog.d/added-18161-derived-reference-repos-settings.md b/changelog.d/added-18161-derived-reference-repos-settings.md new file mode 100644 index 0000000000..2cd5758a6b --- /dev/null +++ b/changelog.d/added-18161-derived-reference-repos-settings.md @@ -0,0 +1 @@ +- Dashboard: Settings → Repos → Reference repos shows entries derived from an accepted request issue as read-only rows with a lock, a `from owner/repo#N` link to the issue, a credential-state chip (`write-tier`, `read-only`, `unresolved`) and the reason; `PUT /api/config/reference-repos` answers 409 to any attempt to edit, remove or replace a derived entry, naming the issue to change instead, while hand-written entries stay editable (#18161, part of #18139). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index fcbff8a4bf..cb5556ae61 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -3684,6 +3684,9 @@ "400": { "description": "Invalid list; nothing was stored" }, + "409": { + "description": "The list edits, removes or replaces an entry derived from a request issue (derived_from); the message names the issue to change instead. Nothing was stored" + }, "500": { "description": "The config file could not be written; the previous list is restored on the proxy and nothing changed" } diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index ef9a63eac6..e43c99dcc6 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -102,7 +102,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:541` | | `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:542` | | `GET` | `/api/config/reference-repos` | Owner only | `project.reference_repos` ([#18136](https://github.com/hivecommons/hive/issues/18136)): `reference_repos` (`{repo, write, write_token_file}` plus `derived_from`, `credential_state`, `credential_reason` on a derived entry; always an array) and `write_kinds` | `pkg/dashboard/api.go:563` | -| `PUT` | `/api/config/reference-repos` | Owner only | Replace `project.reference_repos`; `{"reference_repos": []}` clears it. Refused with a 400 and nothing changed when the config loader would reject the list (bare name, repo also in `project.repos`, duplicate, unknown write kind, token file without write) or when a write entry has no credential boot would accept — the same message boot exits 22 with — or when an entry carries a `write_token_file` the config file does not already hold for that repo (the path is operator-only; a dashboard write opt-in uses the App installation). Entries with `derived_from` ([#18139](https://github.com/hivecommons/hive/issues/18139)) are never stored: they are dropped from the saved list and merged back in memory after the save. 500 when the file could not be written (proxy restored, nothing changed). On success the GitHub proxy's reference policies are replaced without a restart; audited as `config_reference_repos` | `pkg/dashboard/api.go:564` | +| `PUT` | `/api/config/reference-repos` | Owner only | Replace `project.reference_repos`; `{"reference_repos": []}` clears it. Refused with a 400 and nothing changed when the config loader would reject the list (bare name, repo also in `project.repos`, duplicate, unknown write kind, token file without write) or when a write entry has no credential boot would accept — the same message boot exits 22 with — or when an entry carries a `write_token_file` the config file does not already hold for that repo (the path is operator-only; a dashboard write opt-in uses the App installation). Entries with `derived_from` ([#18139](https://github.com/hivecommons/hive/issues/18139)) are the request issue's grant: echo them back as served; changing one's write set, leaving one out, sending one the hive did not derive or adding a hand-written entry for the same repo is a 409 naming the issue to change instead ([#18161](https://github.com/hivecommons/hive/issues/18161)). They are never stored: the saved list holds the hand-written entries and the derived set is merged back in memory after the save. 500 when the file could not be written (proxy restored, nothing changed). On success the GitHub proxy's reference policies are replaced without a restart; audited as `config_reference_repos` | `pkg/dashboard/api.go:564` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:237` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:240` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:246` | diff --git a/src/pkg/dashboard/api_config_reference_repos.go b/src/pkg/dashboard/api_config_reference_repos.go index 42345dedca..b4e89cf8a7 100644 --- a/src/pkg/dashboard/api_config_reference_repos.go +++ b/src/pkg/dashboard/api_config_reference_repos.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sort" "strconv" "strings" @@ -76,6 +77,10 @@ func (s *Server) handleReferenceReposPut(w http.ResponseWriter, r *http.Request) // echoing one back is dropped here; the derived set is merged back in // after the save. prevRepos := cfg.Project.ReferenceRepos + if err := rejectDerivedReferenceRepoEdits(*body.ReferenceRepos, prevRepos); err != nil { + jsonError(w, err.Error(), http.StatusConflict) + return + } next := normalizeReferenceRepos(config.HandWrittenReferenceRepos(*body.ReferenceRepos)) if err := rejectDashboardTokenFiles(next, config.HandWrittenReferenceRepos(prevRepos)); err != nil { jsonError(w, err.Error(), http.StatusBadRequest) @@ -138,6 +143,57 @@ func (s *Server) handleReferenceReposPut(w http.ResponseWriter, r *http.Request) jsonResponse(w, referenceReposSection(cfg)) } +// rejectDerivedReferenceRepoEdits compares the incoming list with the live +// derived set (hivecommons/hive#18161): a derived entry is the request +// issue's grant, so the editor may only echo it back unchanged. Changing its +// write set, leaving it out, sending one the hive did not derive, or adding a +// hand-written entry for the same repo (which would override the grant's +// tier) is a 409 that names the issue to change instead. +func rejectDerivedReferenceRepoEdits(incoming, live []config.ReferenceRepo) error { + key := func(repo string) string { return strings.ToLower(strings.TrimSpace(repo)) } + liveDerived := map[string]config.ReferenceRepo{} + for _, rr := range live { + if rr.Derived() { + liveDerived[key(rr.Repo)] = rr + } + } + conflict := func(rr config.ReferenceRepo, what string) error { + return fmt.Errorf("reference repo %q is derived from request issue %s and cannot be %s here; edit or close that issue instead", rr.Repo, rr.DerivedFrom, what) + } + seen := map[string]bool{} + for _, in := range normalizeReferenceRepos(incoming) { + k := key(in.Repo) + cur, isLive := liveDerived[k] + switch { + case !in.Derived() && isLive: + return conflict(cur, "replaced by a hand-written entry") + case in.Derived() && !isLive: + return fmt.Errorf("reference repo %q is not derived from %s on this hive; a derived entry can only be echoed back as the hive serves it", in.Repo, in.DerivedFrom) + case in.Derived() && (in.DerivedFrom != cur.DerivedFrom || writeSetKey(in.Write) != writeSetKey(cur.Write)): + return conflict(cur, "edited") + } + if in.Derived() { + seen[k] = true + } + } + for k, cur := range liveDerived { + if !seen[k] { + return conflict(cur, "removed") + } + } + return nil +} + +// writeSetKey is the write opt-ins as an order-free comparable string. +func writeSetKey(write []string) string { + kinds := make([]string, 0, len(write)) + for _, w := range write { + kinds = append(kinds, strings.ToLower(strings.TrimSpace(w))) + } + sort.Strings(kinds) + return strings.Join(kinds, "+") +} + // rejectDashboardTokenFiles refuses a write_token_file the dashboard would // introduce or change. The path names a file on the hive host that the // credential check reads and sends to GitHub as a bearer; a hosted hive's diff --git a/src/pkg/dashboard/api_config_reference_repos_test.go b/src/pkg/dashboard/api_config_reference_repos_test.go index e0d860ddc4..a1a939ea40 100644 --- a/src/pkg/dashboard/api_config_reference_repos_test.go +++ b/src/pkg/dashboard/api_config_reference_repos_test.go @@ -296,3 +296,56 @@ func TestReferenceReposPut_DerivedEntriesStrippedAndRemerged(t *testing.T) { t.Fatalf("saved config must hold the hand-written entries:\n%s", data) } } + +// A derived entry (hivecommons/hive#18161) is the request issue's grant: the +// editor may echo it back as served but any edit, removal, replacement by a +// hand-written entry or unknown derived entry is a 409 naming the issue, and +// nothing changes. Hand-written entries stay editable next to it. +func TestReferenceReposPut_DerivedEntriesConflict(t *testing.T) { + s := covApiServer(t) + cfg := s.deps.Config + derived := config.ReferenceRepo{Repo: "acme/docs", Write: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier} + live := []config.ReferenceRepo{{Repo: "keep/me"}, derived} + cfg.Project.ReferenceRepos = append([]config.ReferenceRepo{}, live...) + applied := 0 + s.deps.ApplyReferenceRepos = func(context.Context, []config.ReferenceRepo) error { applied++; return nil } + + echo := map[string]any{"repo": derived.Repo, "write": derived.Write, "derived_from": derived.DerivedFrom} + cases := []struct { + name string + body []map[string]any + }{ + {"removed", []map[string]any{{"repo": "keep/me"}}}, + {"write set edited", []map[string]any{{"repo": "keep/me"}, {"repo": derived.Repo, "write": []string{"issues"}, "derived_from": derived.DerivedFrom}}}, + {"source changed", []map[string]any{{"repo": "keep/me"}, {"repo": derived.Repo, "write": derived.Write, "derived_from": "cncf/techdocs#1"}}}, + {"replaced by a hand-written entry", []map[string]any{{"repo": "keep/me"}, {"repo": "ACME/docs", "write": []string{"issues"}}}}, + {"unknown derived entry", []map[string]any{{"repo": "keep/me"}, echo, {"repo": "zeta/site", "derived_from": "cncf/techdocs#396"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": tc.body}) + if rec.Code != http.StatusConflict { + t.Fatalf("expected 409, got %d: %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "cncf/techdocs#") { + t.Fatalf("409 must name the request issue: %s", rec.Body.String()) + } + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, live) || applied != 0 { + t.Fatalf("a refused PUT changed state: %+v applied=%d", cfg.Project.ReferenceRepos, applied) + } + }) + } + + // Hand-written edits around the echoed derived entry still save. + rec := doPut(s, referenceReposPath, map[string]any{"reference_repos": []map[string]any{{"repo": "keep/me", "write": []string{"issues"}}, echo, {"repo": "new/one"}}}) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + if applied != 1 { + t.Fatalf("proxy hook calls = %d, want 1", applied) + } + want := []config.ReferenceRepo{{Repo: "keep/me", Write: []string{config.ReferenceWriteIssues}}, {Repo: "new/one"}} + if !reflect.DeepEqual(cfg.Project.ReferenceRepos, want) { + t.Fatalf("config holds %+v, want %+v (no remerge hook in this test)", cfg.Project.ReferenceRepos, want) + } +} diff --git a/src/pkg/dashboard/reference_repos_editor.test.js b/src/pkg/dashboard/reference_repos_editor.test.js index db75a30a42..46ff6d0369 100644 --- a/src/pkg/dashboard/reference_repos_editor.test.js +++ b/src/pkg/dashboard/reference_repos_editor.test.js @@ -3,7 +3,9 @@ // and a remove button, shows an operator-set write_token_file as read-only // text (never an input: the path is file-only), carries derived_from through // to the PUT body so the server can drop derived entries, and shows the -// server's refusal verbatim without losing the edits. +// server's refusal verbatim without losing the edits. Derived entries +// (hivecommons/hive#18161) are read-only rows with a lock, a link to the +// request issue and a credential-state chip; a 409 is also toasted. // Run: node --test src/pkg/dashboard/*.test.js const test = require('node:test'); const assert = require('node:assert'); @@ -46,7 +48,7 @@ const makeDoc = () => { return el; }; doc.createElement = makeEl; - doc.createTextNode = text => ({ tag: '#text', children: [], textContent: String(text) }); + doc.createTextNode = text => ({ tag: '#text', children: [], className: '', textContent: String(text) }); doc.getElementById = () => null; return doc; }; @@ -59,15 +61,15 @@ const setup = ({ entries, kinds = ['issues', 'prs'], put }) => { calls.push({ url, opts, body: JSON.parse(opts.body) }); return put(calls[calls.length - 1]); }; - const render = new Function('document', 'fetch', 'showToast', '_inceptionAuthHeaders', 'saveErrorMessage', - fnSrc('referenceRepoNode') + '\n' + fnSrc('referenceRepoYamlHint') + '\n' + fnSrc('renderReferenceReposPanel') + '\nreturn renderReferenceReposPanel;')( - doc, fetchStub, (m, k) => toasts.push([m, k]), () => ({}), async res => res.errorMessage); + const render = new Function('document', 'window', 'fetch', 'showToast', '_inceptionAuthHeaders', 'saveErrorMessage', + fnSrc('referenceRepoNode') + '\n' + fnSrc('referenceRepoYamlHint') + '\n' + fnSrc('referenceRepoDerivedLink') + '\n' + fnSrc('renderReferenceReposPanel') + '\nreturn renderReferenceReposPanel;')( + doc, { _githubBaseUrl: 'https://github.example.com/' }, fetchStub, (m, k) => toasts.push([m, k]), () => ({}), async res => res.errorMessage); const host = doc.createElement('div'); render(host, entries, kinds); return { doc, host, toasts, calls }; }; -const rows = host => host.find(n => n.className === 'config-list-item config-repo-row'); +const rows = host => host.find(n => String(n.className).startsWith('config-list-item config-repo-row')); const inputs = (node, type) => node.find(n => n.tag === 'input' && n.type === type); const buttons = (node, text) => node.find(n => n.tag === 'button' && n.textContent === text); @@ -148,3 +150,54 @@ test('add, toggle and remove edit the list; a refused save shows the server mess assert.strictEqual(rows(host).length, 1, 'edits survive a refused save'); assert.strictEqual(buttons(host, 'Save reference repos')[0].disabled, false); }); + +test('a derived entry is a read-only row: lock, source issue link, state chip and reason, no controls', () => { + const { host } = setup({ + entries: [ + { repo: 'acme/docs', write: ['issues', 'prs'], derived_from: 'cncf/techdocs#395', credential_state: 'write-tier' }, + { repo: 'zeta/site', derived_from: 'cncf/techdocs#395', credential_state: 'read-only', credential_reason: 'no App installation on zeta' }, + { repo: 'hand/written', write: ['issues'] }, + ], + put: () => { throw new Error('no save'); }, + }); + const r = rows(host); + assert.strictEqual(r.length, 3); + const [docs, zeta, hand] = r; + assert.ok(docs.className.includes('cfg-reference-repo-derived')); + assert.strictEqual(docs.find(n => n.className === 'cfg-reference-repo-lock').length, 1); + assert.strictEqual(inputs(docs, 'checkbox').length, 0, 'no write checkboxes on a derived row'); + assert.strictEqual(buttons(docs, '\u2715').length, 0, 'no remove button on a derived row'); + const link = docs.find(n => n.tag === 'a')[0]; + assert.strictEqual(link.textContent, 'from cncf/techdocs#395'); + assert.strictEqual(link.href, 'https://github.example.com/cncf/techdocs/issues/395'); + assert.strictEqual(link.target, '_blank'); + const chip = docs.find(n => n.className.includes('cfg-reference-repo-state'))[0]; + assert.strictEqual(chip.textContent, 'issues+prs \u00b7 write-tier'); + assert.ok(chip.className.includes('cfg-reference-repo-state-write-tier')); + assert.strictEqual(docs.find(n => n.className.includes('cfg-reference-repo-reason')).length, 0); + + const zchip = zeta.find(n => n.className.includes('cfg-reference-repo-state'))[0]; + assert.strictEqual(zchip.textContent, 'read-only \u00b7 read-only'); + assert.strictEqual(zchip.title, 'no App installation on zeta'); + assert.strictEqual(zeta.find(n => n.className.includes('cfg-reference-repo-reason'))[0].textContent, 'no App installation on zeta'); + + assert.ok(!hand.className.includes('cfg-reference-repo-derived')); + assert.strictEqual(inputs(hand, 'checkbox').length, 2, 'hand-written rows keep their controls'); + assert.strictEqual(buttons(hand, '\u2715').length, 1); +}); + +test('a 409 on save is toasted as an error and shown in the status line; the list is kept', async () => { + const message = 'reference repo "acme/docs" is derived from request issue cncf/techdocs#395 and cannot be removed here; edit or close that issue instead'; + const { host, calls, toasts } = setup({ + entries: [{ repo: 'acme/docs', write: ['issues', 'prs'], derived_from: 'cncf/techdocs#395', credential_state: 'write-tier' }, { repo: 'hand/written' }], + put: () => ({ ok: false, status: 409, errorMessage: message }), + }); + await buttons(host, 'Save reference repos')[0].fire('click')[0]; + assert.deepStrictEqual(calls[0].body.reference_repos, [ + { repo: 'acme/docs', write: ['issues', 'prs'], derived_from: 'cncf/techdocs#395' }, + { repo: 'hand/written' }, + ]); + assert.deepStrictEqual(toasts, [[message, 'error']]); + assert.strictEqual(host.find(n => n.className === 'fs-sm text-danger')[0].textContent, message); + assert.strictEqual(rows(host).length, 2); +}); diff --git a/src/pkg/dashboard/static-src/css/config-dialog.css b/src/pkg/dashboard/static-src/css/config-dialog.css index 8dc476ea89..6a5b4c04ba 100644 --- a/src/pkg/dashboard/static-src/css/config-dialog.css +++ b/src/pkg/dashboard/static-src/css/config-dialog.css @@ -496,6 +496,13 @@ .config-repo-row { flex-wrap: wrap; } .config-repo-name { flex: 1 1 14rem; min-width: 0; overflow-wrap: anywhere; } .config-repo-row .rt-check { white-space: normal; min-width: 0; } + /* Derived reference repo rows (#18161): read-only, source link and state chip. */ + .cfg-reference-repo-derived { opacity: 0.92; } + .cfg-reference-repo-lock { font-size: var(--fs-xs); } + .cfg-reference-repo-source { font-size: var(--fs-xs); } + .cfg-reference-repo-state-write-tier { color: var(--success, var(--muted)); } + .cfg-reference-repo-state-read-only, .cfg-reference-repo-state-unresolved { color: var(--warning, var(--muted)); } + .cfg-reference-repo-reason { flex-basis: 100%; } .config-list-item .remove-item { margin-left: auto; background: none; border: none; color: var(--red); cursor: pointer; font-size: 0.9rem; diff --git a/src/pkg/dashboard/static-src/js/config-governor.js b/src/pkg/dashboard/static-src/js/config-governor.js index 0ee2364798..1099393e14 100644 --- a/src/pkg/dashboard/static-src/js/config-governor.js +++ b/src/pkg/dashboard/static-src/js/config-governor.js @@ -1227,6 +1227,21 @@ return node; } + /* referenceRepoDerivedLink is the "from owner/repo#N" link to the request + issue a derived entry came from (#18161), on the hive's GitHub host. A + ref that does not parse is shown as text. */ + function referenceRepoDerivedLink(ref) { + const m = /^([^\/#\s]+\/[^\/#\s]+)#(\d+)$/.exec(String(ref || '').trim()); + if (!m) return referenceRepoNode('span', 'text-muted fs-sm cfg-reference-repo-source', 'from ' + ref); + const base = ((typeof window !== 'undefined' && window._githubBaseUrl) || 'https://github.com').replace(/\/+$/, ''); + const a = referenceRepoNode('a', 'fs-sm cfg-reference-repo-source', 'from ' + m[1] + '#' + m[2]); + a.href = base + '/' + m[1] + '/issues/' + m[2]; + a.target = '_blank'; + a.rel = 'noopener'; + a.title = 'The accepted request issue this entry is derived from'; + return a; + } + /* referenceRepoYamlHint is yamlHint() as a DOM node. */ function referenceRepoYamlHint(path) { const hint = referenceRepoNode('span', 'settings-yaml-hint', 'YAML '); @@ -1241,7 +1256,7 @@ /* derived_from is carried through so the server can tell a derived entry (rebuilt from its request issue, never saved) from an operator's. */ - entries = entries.map(e => ({ repo: e.repo || '', write: (e.write || []).slice(), write_token_file: e.write_token_file || '', derived_from: e.derived_from || '' })); + entries = entries.map(e => ({ repo: e.repo || '', write: (e.write || []).slice(), write_token_file: e.write_token_file || '', derived_from: e.derived_from || '', credential_state: e.credential_state || '', credential_reason: e.credential_reason || '' })); host.textContent = ''; host.appendChild(referenceRepoNode('p', 'config-help', 'Read-only by default: agents clone and read these repos and are never told they are out of scope, but every write is refused. ' + @@ -1258,6 +1273,25 @@ if (!entries.length) list.appendChild(referenceRepoNode('div', 'fs-sm text-muted mb-3', 'No reference repos configured')); entries.forEach((entry, i) => { const row = referenceRepoNode('div', 'config-list-item config-repo-row'); + if (entry.derived_from) { + /* Derived entry (#18161): the request issue's grant, shown but + never edited here. A lock, the source issue link, the credential + state chip and its reason; no checkboxes, no remove button. */ + row.className += ' cfg-reference-repo-derived'; + const lock = referenceRepoNode('span', 'cfg-reference-repo-lock', '🔒'); + lock.title = 'Derived from ' + entry.derived_from + ': edit or close that issue to change it.'; + row.appendChild(lock); + row.appendChild(referenceRepoNode('span', 'config-repo-name', entry.repo)); + row.appendChild(referenceRepoDerivedLink(entry.derived_from)); + const state = entry.credential_state || 'read-only'; + const chip = referenceRepoNode('span', 'rt-inherit cfg-reference-repo-state cfg-reference-repo-state-' + state, + (entry.write.length ? entry.write.join('+') : 'read-only') + ' · ' + state); + chip.title = entry.credential_reason || (state === 'write-tier' ? 'The GitHub App is installed on the repo\'s org; issues and PRs may be opened there.' : 'No credential covers this repo; served read-only.'); + row.appendChild(chip); + if (entry.credential_reason) row.appendChild(referenceRepoNode('span', 'text-muted fs-sm cfg-reference-repo-reason', entry.credential_reason)); + list.appendChild(row); + return; + } row.appendChild(referenceRepoNode('span', 'config-repo-name', entry.repo)); kinds.forEach(kind => { const label = referenceRepoNode('label', 'rt-check'); @@ -1345,7 +1379,13 @@ headers: _inceptionAuthHeaders(), body: JSON.stringify({ reference_repos: body }) }); - if (!res.ok) throw new Error(await saveErrorMessage(res)); + if (!res.ok) { + const msg = await saveErrorMessage(res); + /* 409: the list touched a derived entry (#18161); the message + names the request issue to change instead. */ + if (res.status === 409) showToast(msg, 'error'); + throw new Error(msg); + } const d = await res.json(); renderReferenceReposPanel(host, d.reference_repos || [], d.write_kinds || kinds); showToast('Reference repos saved', 'success'); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 48f2850d8c..9aff2831a4 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5025,6 +5025,13 @@ .config-repo-row { flex-wrap: wrap; } .config-repo-name { flex: 1 1 14rem; min-width: 0; overflow-wrap: anywhere; } .config-repo-row .rt-check { white-space: normal; min-width: 0; } + /* Derived reference repo rows (#18161): read-only, source link and state chip. */ + .cfg-reference-repo-derived { opacity: 0.92; } + .cfg-reference-repo-lock { font-size: var(--fs-xs); } + .cfg-reference-repo-source { font-size: var(--fs-xs); } + .cfg-reference-repo-state-write-tier { color: var(--success, var(--muted)); } + .cfg-reference-repo-state-read-only, .cfg-reference-repo-state-unresolved { color: var(--warning, var(--muted)); } + .cfg-reference-repo-reason { flex-basis: 100%; } .config-list-item .remove-item { margin-left: auto; background: none; border: none; color: var(--red); cursor: pointer; font-size: 0.9rem; @@ -64511,6 +64518,21 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> return node; } + /* referenceRepoDerivedLink is the "from owner/repo#N" link to the request + issue a derived entry came from (#18161), on the hive's GitHub host. A + ref that does not parse is shown as text. */ + function referenceRepoDerivedLink(ref) { + const m = /^([^\/#\s]+\/[^\/#\s]+)#(\d+)$/.exec(String(ref || '').trim()); + if (!m) return referenceRepoNode('span', 'text-muted fs-sm cfg-reference-repo-source', 'from ' + ref); + const base = ((typeof window !== 'undefined' && window._githubBaseUrl) || 'https://github.com').replace(/\/+$/, ''); + const a = referenceRepoNode('a', 'fs-sm cfg-reference-repo-source', 'from ' + m[1] + '#' + m[2]); + a.href = base + '/' + m[1] + '/issues/' + m[2]; + a.target = '_blank'; + a.rel = 'noopener'; + a.title = 'The accepted request issue this entry is derived from'; + return a; + } + /* referenceRepoYamlHint is yamlHint() as a DOM node. */ function referenceRepoYamlHint(path) { const hint = referenceRepoNode('span', 'settings-yaml-hint', 'YAML '); @@ -64525,7 +64547,7 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> /* derived_from is carried through so the server can tell a derived entry (rebuilt from its request issue, never saved) from an operator's. */ - entries = entries.map(e => ({ repo: e.repo || '', write: (e.write || []).slice(), write_token_file: e.write_token_file || '', derived_from: e.derived_from || '' })); + entries = entries.map(e => ({ repo: e.repo || '', write: (e.write || []).slice(), write_token_file: e.write_token_file || '', derived_from: e.derived_from || '', credential_state: e.credential_state || '', credential_reason: e.credential_reason || '' })); host.textContent = ''; host.appendChild(referenceRepoNode('p', 'config-help', 'Read-only by default: agents clone and read these repos and are never told they are out of scope, but every write is refused. ' + @@ -64542,6 +64564,25 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> if (!entries.length) list.appendChild(referenceRepoNode('div', 'fs-sm text-muted mb-3', 'No reference repos configured')); entries.forEach((entry, i) => { const row = referenceRepoNode('div', 'config-list-item config-repo-row'); + if (entry.derived_from) { + /* Derived entry (#18161): the request issue's grant, shown but + never edited here. A lock, the source issue link, the credential + state chip and its reason; no checkboxes, no remove button. */ + row.className += ' cfg-reference-repo-derived'; + const lock = referenceRepoNode('span', 'cfg-reference-repo-lock', '🔒'); + lock.title = 'Derived from ' + entry.derived_from + ': edit or close that issue to change it.'; + row.appendChild(lock); + row.appendChild(referenceRepoNode('span', 'config-repo-name', entry.repo)); + row.appendChild(referenceRepoDerivedLink(entry.derived_from)); + const state = entry.credential_state || 'read-only'; + const chip = referenceRepoNode('span', 'rt-inherit cfg-reference-repo-state cfg-reference-repo-state-' + state, + (entry.write.length ? entry.write.join('+') : 'read-only') + ' · ' + state); + chip.title = entry.credential_reason || (state === 'write-tier' ? 'The GitHub App is installed on the repo\'s org; issues and PRs may be opened there.' : 'No credential covers this repo; served read-only.'); + row.appendChild(chip); + if (entry.credential_reason) row.appendChild(referenceRepoNode('span', 'text-muted fs-sm cfg-reference-repo-reason', entry.credential_reason)); + list.appendChild(row); + return; + } row.appendChild(referenceRepoNode('span', 'config-repo-name', entry.repo)); kinds.forEach(kind => { const label = referenceRepoNode('label', 'rt-check'); @@ -64629,7 +64670,13 @@ <h4 class="rt-h4">Clanker-requested PRs</h4> headers: _inceptionAuthHeaders(), body: JSON.stringify({ reference_repos: body }) }); - if (!res.ok) throw new Error(await saveErrorMessage(res)); + if (!res.ok) { + const msg = await saveErrorMessage(res); + /* 409: the list touched a derived entry (#18161); the message + names the request issue to change instead. */ + if (res.status === 409) showToast(msg, 'error'); + throw new Error(msg); + } const d = await res.json(); renderReferenceReposPanel(host, d.reference_repos || [], d.write_kinds || kinds); showToast('Reference repos saved', 'success'); From f9c6368040e3967cfdaad8639bc2f91fef197108 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 23:02:06 -0500 Subject: [PATCH 213/223] =?UTF-8?q?=F0=9F=90=9B=20proxy:=20reload=20the=20?= =?UTF-8?q?UID=20map=20on=20a=20lookup=20miss=20so=20runtime-added=20agent?= =?UTF-8?q?s=20are=20attributed=20and=20isolated=20(#18185)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy read /var/run/hive/uid-map.json once at construction, so an agent the manager added later (AddAgent, ReconcileAgents) was refused as uid_not_mapped from its own socket, and LookupByName returned 0 for it — letting a proxy-UID caller claim its name past the #18133 check until the hive restarted. Every proxy read of the map now goes through loadedUIDMap, and a UID or name lookup that misses re-reads the file once if its stat stamp changed (the file, not the manager's in-memory map, so sidecar mode is covered). A map absent at boot stays absent; a malformed rewrite keeps the old map. Fixes #18184 Hive-Run: hivecommons/hive#18184 Hive-Plan: issue-18184 fix (reload uid-map.json in proxy) Hive-Spec: hivecommons/hive#18184#fix Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- ...rity-18184-proxy-uid-map-runtime-agents.md | 1 + src/crux/proxy/github_proxy.go | 20 ++- src/crux/proxy/github_proxy_connect.go | 6 +- src/crux/proxy/github_proxy_identity.go | 8 +- src/crux/proxy/github_proxy_inference.go | 4 +- src/crux/proxy/github_proxy_uidmap.go | 102 +++++++++++++ src/crux/proxy/github_proxy_uidmap_test.go | 137 ++++++++++++++++++ src/crux/proxy/inference_refusal.go | 10 +- 8 files changed, 272 insertions(+), 16 deletions(-) create mode 100644 changelog.d/security-18184-proxy-uid-map-runtime-agents.md create mode 100644 src/crux/proxy/github_proxy_uidmap.go create mode 100644 src/crux/proxy/github_proxy_uidmap_test.go diff --git a/changelog.d/security-18184-proxy-uid-map-runtime-agents.md b/changelog.d/security-18184-proxy-uid-map-runtime-agents.md new file mode 100644 index 0000000000..effd5ec377 --- /dev/null +++ b/changelog.d/security-18184-proxy-uid-map-runtime-agents.md @@ -0,0 +1 @@ +- The egress proxy now reloads `uid-map.json` when a UID or agent name is not in its copy, so an agent added at runtime is identified from its own socket and protected by the proxy-UID name-claim check ([#18133](https://github.com/hivecommons/hive/issues/18133)) without restarting the hive ([#18184](https://github.com/hivecommons/hive/issues/18184)). diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 5b616e62b2..008dfdd63c 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -84,8 +84,16 @@ type GitHubProxy struct { caCert tls.Certificate caX509 *x509.Certificate logger *slog.Logger - uidMap *agent.UIDMap - allowedRepos map[string]bool + // uidMap is read through loadedUIDMap and swapped only by refreshUIDMap + // under uidMapMu (#18184): agents added at runtime land in uid-map.json + // after construction. uidMapPath/uidMapInfo are what the refresh compares + // against; an empty path (no map at boot, or a test literal) never reloads. + uidMap *agent.UIDMap + uidMapMu sync.RWMutex + uidMapReloadMu sync.Mutex + uidMapPath string + uidMapInfo os.FileInfo + allowedRepos map[string]bool // Reference repos (#18015): study-only repositories outside allowedRepos, // read-only by default with a per-entry issues/prs opt-in. Reads to them // are forwarded without the hive's credential; writes are gated by @@ -279,8 +287,14 @@ func NewGitHubProxyEphemeral(logger *slog.Logger, org string, repos []string) (* func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logger *slog.Logger, org string, repos []string) *GitHubProxy { var uidMap *agent.UIDMap + var uidMapPath string + var uidMapInfo os.FileInfo + // Stat before load: a write landing in between leaves an older stamp, so + // the next lookup miss reloads rather than missing the change. + statInfo, _ := os.Stat(agent.UIDMapPath) if loaded, loadErr := agent.LoadUIDMap(agent.UIDMapPath); loadErr == nil { uidMap = loaded + uidMapPath, uidMapInfo = agent.UIDMapPath, statInfo logger.Info("proxy loaded UID map", "agents", len(uidMap.Agents), "iptables", uidMap.IptablesActive) } @@ -323,6 +337,8 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg caX509: caX509, logger: logger, uidMap: uidMap, + uidMapPath: uidMapPath, + uidMapInfo: uidMapInfo, proxyAdvisoryOK: advisoryOK, allowPrivateForward: allowPrivateForward, injectGHAuth: injectGHAuth, diff --git a/src/crux/proxy/github_proxy_connect.go b/src/crux/proxy/github_proxy_connect.go index 6cb72dd604..01c101a1d2 100644 --- a/src/crux/proxy/github_proxy_connect.go +++ b/src/crux/proxy/github_proxy_connect.go @@ -237,7 +237,7 @@ func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { // an allocated agent UID) is named internalCallerName so a hive-originated // write is attributed rather than mistaken for a UID-attribution failure. func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { - if p.uidMap == nil { + if p.loadedUIDMap() == nil { return "" } _, portStr, splitErr := net.SplitHostPort(conn.RemoteAddr().String()) @@ -257,7 +257,7 @@ func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { if lookupErr != nil { return "" } - if name := p.uidMap.LookupByUID(uid); name != "" { + if name := p.lookupAgentByUID(uid); name != "" { return name } // The hive's own control plane is not an agent, so the lookup above @@ -266,7 +266,7 @@ func (p *GitHubProxy) attributeTransparentConn(conn net.Conn) string { // like an unattributable agent, which the mode gate blocks. Name it // explicitly so it is attributed rather than mistaken for a // UID-attribution failure. - if p.uidMap.IsInternalUID(uid) { + if p.loadedUIDMap().IsInternalUID(uid) { return internalCallerName } return "" diff --git a/src/crux/proxy/github_proxy_identity.go b/src/crux/proxy/github_proxy_identity.go index 03d95e5366..a024304eac 100644 --- a/src/crux/proxy/github_proxy_identity.go +++ b/src/crux/proxy/github_proxy_identity.go @@ -16,7 +16,7 @@ import ( // including native binaries that don't send Proxy-Authorization); see // fallbackAgentName for what happens when that fails. func (p *GitHubProxy) identifyAgentFromReq(r *http.Request) string { - if p.uidMap != nil { + if p.loadedUIDMap() != nil { if name := p.identifyAgentByUID(r.RemoteAddr); name != "" { return name } @@ -33,7 +33,7 @@ func (p *GitHubProxy) identifyAgentFromReq(r *http.Request) string { // functional" mode worth that trade — an unidentified caller is refused. // Returns "" when no UID map was loaded or the connection's UID is unmapped. func (p *GitHubProxy) IdentifyAgentByUID(r *http.Request) string { - if p.uidMap == nil || r == nil { + if p.loadedUIDMap() == nil || r == nil { return "" } return p.identifyAgentByUID(r.RemoteAddr) @@ -47,7 +47,7 @@ func (p *GitHubProxy) IdentifyAgentByUID(r *http.Request) string { // falls back to ADVISORY — downgrading every write-capable agent to read-only. // Taking the peer address from the connection itself removes that trap. func (p *GitHubProxy) identifyAgentFromConn(conn net.Conn, r *http.Request) string { - if p.uidMap != nil && conn != nil { + if p.loadedUIDMap() != nil && conn != nil { if name := p.identifyAgentByUID(conn.RemoteAddr().String()); name != "" { return name } @@ -101,7 +101,7 @@ func (p *GitHubProxy) identifyAgentByUID(remoteAddr string) string { if err != nil { return "" } - return p.uidMap.LookupByUID(uid) + return p.lookupAgentByUID(uid) } // uidForRemoteAddr resolves the UID of the process that owns the local socket diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 2a3f500ffb..fba6ad30a6 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -73,7 +73,7 @@ const inferenceAPIKeyPrefix = "sk-hive-" // request must be refused. func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) - if p.uidMap == nil { + if p.loadedUIDMap() == nil { if p.proxyAdvisoryOK { return claimed, true } @@ -81,7 +81,7 @@ func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { } uid, err := uidForRemoteAddr(r.RemoteAddr) if err == nil { - if name := p.uidMap.LookupByUID(uid); name != "" { + if name := p.lookupAgentByUID(uid); name != "" { if claimed != "" && claimed != name { p.logger.Warn("inference identity: x-api-key names another agent; using the socket owner", "agent", name, "claimed", claimed, "uid", uid) diff --git a/src/crux/proxy/github_proxy_uidmap.go b/src/crux/proxy/github_proxy_uidmap.go new file mode 100644 index 0000000000..7c934b5f6f --- /dev/null +++ b/src/crux/proxy/github_proxy_uidmap.go @@ -0,0 +1,102 @@ +package proxy + +import ( + "os" + + "github.com/hivecommons/hive/crux/agent" +) + +// loadedUIDMap returns the UID map the proxy currently attributes callers +// with, or nil when none was loaded at construction. Every reader goes through +// here because refreshUIDMap may swap the pointer while requests are in +// flight. +func (p *GitHubProxy) loadedUIDMap() *agent.UIDMap { + p.uidMapMu.RLock() + defer p.uidMapMu.RUnlock() + return p.uidMap +} + +// refreshUIDMap re-reads uid-map.json when it changed since the proxy last +// loaded it and returns the new map, or nil when nothing changed or no reload +// is possible. +// +// WHY (#18184). The manager allocates a UID for an agent added at runtime +// (AddAgent, ReconcileAgents) and saves the map, but the proxy used to read the +// file once at construction. Until a restart such an agent's own socket was +// refused as uid_not_mapped, and — worse — LookupByName returned 0 for it, so +// the #18133 check that stops a proxy-UID caller from claiming a UID-isolated +// agent's name accepted the claim. The file, not the manager's in-memory map, +// is the source here because in sidecar mode the manager runs in another +// process. +// +// The callers reload only on a lookup MISS, so a known agent costs nothing +// and an unknown UID or name costs one stat. A map absent at construction +// stays absent (uidMapPath is empty): that is the no-entrypoint deployment +// whose ADVISORY semantics must not flip mid-run. An unreadable or malformed +// file keeps the previous map; its stamp is still recorded so a bad file is +// warned about once per change rather than on every request. +func (p *GitHubProxy) refreshUIDMap() *agent.UIDMap { + if p.uidMapPath == "" { + return nil + } + p.uidMapReloadMu.Lock() + defer p.uidMapReloadMu.Unlock() + + info, err := os.Stat(p.uidMapPath) + if err != nil { + return nil + } + if prev := p.uidMapInfo; prev != nil && os.SameFile(prev, info) && + prev.ModTime().Equal(info.ModTime()) && prev.Size() == info.Size() { + return nil + } + p.uidMapInfo = info + loaded, err := agent.LoadUIDMap(p.uidMapPath) + if err != nil { + if p.logger != nil { + p.logger.Warn("proxy: UID map changed but could not be reloaded; keeping the previous map", "path", p.uidMapPath, "error", err) + } + return nil + } + p.uidMapMu.Lock() + p.uidMap = loaded + p.uidMapMu.Unlock() + if p.logger != nil { + p.logger.Info("proxy reloaded UID map", "path", p.uidMapPath, "agents", len(loaded.Agents)) + } + return loaded +} + +// lookupAgentByUID names the agent that owns uid, reloading the UID map once +// when uid is not in it so an agent added at runtime is attributed without a +// restart. Returns "" when no map is loaded or uid belongs to no agent. +func (p *GitHubProxy) lookupAgentByUID(uid int) string { + m := p.loadedUIDMap() + if m == nil { + return "" + } + if name := m.LookupByUID(uid); name != "" { + return name + } + if fresh := p.refreshUIDMap(); fresh != nil { + return fresh.LookupByUID(uid) + } + return "" +} + +// lookupUIDByName returns the per-agent UID allocated to name, reloading the +// UID map once on a miss (see lookupAgentByUID). Returns 0 when no map is +// loaded or name has no per-agent UID. +func (p *GitHubProxy) lookupUIDByName(name string) int { + m := p.loadedUIDMap() + if m == nil { + return 0 + } + if uid := m.LookupByName(name); uid > 0 { + return uid + } + if fresh := p.refreshUIDMap(); fresh != nil { + return fresh.LookupByName(name) + } + return 0 +} diff --git a/src/crux/proxy/github_proxy_uidmap_test.go b/src/crux/proxy/github_proxy_uidmap_test.go new file mode 100644 index 0000000000..9dd66db663 --- /dev/null +++ b/src/crux/proxy/github_proxy_uidmap_test.go @@ -0,0 +1,137 @@ +package proxy + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +// bootUIDMap writes a uid-map.json holding "quality" at 2002 with proxy UID +// 1001, points agent.UIDMapPath at it, and builds the proxy the way boot does, +// so the proxy's copy is the one loaded at construction. +func bootUIDMap(t *testing.T, logOut io.Writer) (*GitHubProxy, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "uid-map.json") + m := agent.NewUIDMap() + m.Agents["quality"] = 2002 + if err := m.Save(path); err != nil { + t.Fatal(err) + } + old := agent.UIDMapPath + agent.UIDMapPath = path + t.Cleanup(func() { agent.UIDMapPath = old }) + + p, err := NewGitHubProxyEphemeral(slog.New(slog.NewTextHandler(logOut, nil)), "org", []string{"repo"}) + if err != nil { + t.Fatal(err) + } + if p.loadedUIDMap() == nil { + t.Fatal("proxy did not load the boot UID map") + } + return p, path +} + +// addAgentAtRuntime does what Manager.AddAgent / ReconcileAgents do for a new +// agent after boot: allocate the next UID and save the map to disk. +func addAgentAtRuntime(t *testing.T, path, name string) int { + t.Helper() + m, err := agent.LoadUIDMap(path) + if err != nil { + t.Fatal(err) + } + uid := m.AllocateUID(name) + if err := m.Save(path); err != nil { + t.Fatal(err) + } + return uid +} + +// TestUIDMap_RuntimeAddedAgentOwnSocketAccepted pins #18184: an agent the +// manager adds after the proxy was built is identified from its own socket +// without a restart, instead of being refused as uid_not_mapped. +func TestUIDMap_RuntimeAddedAgentOwnSocketAccepted(t *testing.T) { + var buf strings.Builder + p, path := bootUIDMap(t, &buf) + uid := addAgentAtRuntime(t, path, "scanner") + if uid != 2003 { + t.Fatalf("allocated uid = %d, want 2003", uid) + } + fakeProcNetTCPUID(t, uid) + + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "")); !ok || name != "scanner" { + t.Fatalf("runtime-added agent's own socket: got (%q,%v), want (scanner,true)", name, ok) + } + if got := p.identifyAgentByUID("127.0.0.1:43210"); got != "scanner" { + t.Fatalf("identifyAgentByUID = %q, want scanner", got) + } + if strings.Contains(buf.String(), "uid_not_mapped") { + t.Fatalf("runtime-added agent refused as uid_not_mapped: %s", buf.String()) + } +} + +// TestUIDMap_ProxyUIDClaimOnRuntimeAddedAgentRefused pins #18184 against the +// #18133 invariant: once an agent added at runtime owns a UID, a proxy-UID +// caller claiming its name is refused with proxy_uid_claims_isolated_agent +// rather than accepted because the proxy's map was stale. +func TestUIDMap_ProxyUIDClaimOnRuntimeAddedAgentRefused(t *testing.T) { + var buf strings.Builder + p, path := bootUIDMap(t, &buf) + p.inference = newInferenceRouter() + addAgentAtRuntime(t, path, "scanner") + p.inference.Set("scanner", &InferenceRoute{Backend: "vllm", Endpoint: "http://127.0.0.1:1", Model: "m", APIKey: "victim-secret"}) + + // The name check alone must see the new agent, independent of any UID + // lookup having refreshed the map first. + if !p.sharedUIDClaimsIsolatedAgent(1001, "scanner") { + t.Fatal("sharedUIDClaimsIsolatedAgent did not see the runtime-added agent") + } + + fakeProcNetTCPUID(t, 1001) + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "scanner")); ok || name != "" { + t.Fatalf("proxy uid claiming runtime-added agent: got (%q,%v), want refused", name, ok) + } + rec := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(rec, translatorReq("127.0.0.1:43210", "scanner")) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401; body %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(buf.String(), "reason=proxy_uid_claims_isolated_agent") { + t.Errorf("refusal log %q missing reason=proxy_uid_claims_isolated_agent", buf.String()) + } + if strings.Contains(buf.String(), "victim-secret") { + t.Errorf("log %q leaked the victim's route key", buf.String()) + } +} + +// TestUIDMap_RefreshOnlyWhenFileChanges pins the reload discipline: an +// unchanged file is not re-read, a malformed rewrite keeps the previous map, +// and a map absent at construction never appears mid-run. +func TestUIDMap_RefreshOnlyWhenFileChanges(t *testing.T) { + p, path := bootUIDMap(t, io.Discard) + before := p.loadedUIDMap() + if p.refreshUIDMap() != nil || p.loadedUIDMap() != before { + t.Fatal("unchanged uid-map.json was reloaded") + } + + if err := os.WriteFile(path, []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + if p.refreshUIDMap() != nil || p.loadedUIDMap() != before { + t.Fatal("malformed uid-map.json replaced the loaded map") + } + if got := p.lookupUIDByName("quality"); got != 2002 { + t.Fatalf("after malformed rewrite, quality uid = %d, want 2002", got) + } + + noMap := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + if noMap.refreshUIDMap() != nil || noMap.lookupAgentByUID(2002) != "" || noMap.loadedUIDMap() != nil { + t.Fatal("proxy with no boot UID map loaded one at runtime") + } +} diff --git a/src/crux/proxy/inference_refusal.go b/src/crux/proxy/inference_refusal.go index 747c0d96f6..754b4c59eb 100644 --- a/src/crux/proxy/inference_refusal.go +++ b/src/crux/proxy/inference_refusal.go @@ -9,7 +9,7 @@ import ( // for the translator's purposes: root always, and the proxy UID only when the // proxy cannot relay to private addresses (see translatorAgentName). func (p *GitHubProxy) internalTranslatorUID(uid int) bool { - if !p.uidMap.IsInternalUID(uid) { + if !p.loadedUIDMap().IsInternalUID(uid) { return false } return uid == 0 || !p.allowPrivateForward @@ -24,7 +24,7 @@ func (p *GitHubProxy) internalTranslatorUID(uid int) bool { // the control plane and is not subject to this check, and other unmapped UIDs // keep the HIVE_PROXY_ADVISORY_OK semantics the GitHub path documents. func (p *GitHubProxy) sharedUIDClaimsIsolatedAgent(uid int, claimed string) bool { - return uid != 0 && uid == p.uidMap.ProxyUID && claimed != "" && p.uidMap.LookupByName(claimed) > 0 + return uid != 0 && uid == p.loadedUIDMap().ProxyUID && claimed != "" && p.lookupUIDByName(claimed) > 0 } // translatorRefusalAttrs are the log attributes for a refused translator @@ -37,16 +37,16 @@ func (p *GitHubProxy) sharedUIDClaimsIsolatedAgent(uid int, claimed string) bool func (p *GitHubProxy) translatorRefusalAttrs(r *http.Request) []any { attrs := []any{"remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path} switch { - case p.uidMap == nil: + case p.loadedUIDMap() == nil: attrs = append(attrs, "reason", "no_uid_map") default: uid, err := uidForRemoteAddr(r.RemoteAddr) claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) if err != nil { attrs = append(attrs, "reason", "socket_owner_unresolved", "error", err.Error()) - } else if p.uidMap.IsInternalUID(uid) && p.allowPrivateForward { + } else if p.loadedUIDMap().IsInternalUID(uid) && p.allowPrivateForward { attrs = append(attrs, "reason", "proxy_uid_private_forward", "uid", uid) - } else if p.uidMap.IsInternalUID(uid) && p.sharedUIDClaimsIsolatedAgent(uid, claimed) { + } else if p.loadedUIDMap().IsInternalUID(uid) && p.sharedUIDClaimsIsolatedAgent(uid, claimed) { attrs = append(attrs, "reason", "proxy_uid_claims_isolated_agent", "uid", uid, "claimed", claimed) } else { attrs = append(attrs, "reason", "uid_not_mapped", "uid", uid) From bbbf4282a81f018cf0aa1000d8505f1de684ec49 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 04:11:26 +0000 Subject: [PATCH 214/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.255.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.255.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 11 +++++++++++ ...-18161-repos-card-shows-derived-reference-repos.md | 1 - changelog.d/added-reference-repos-editor.md | 1 - .../fixed-18166-selfservice-docs-double-hyphen-id.md | 1 - 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/added-18161-repos-card-shows-derived-reference-repos.md delete mode 100644 changelog.d/added-reference-repos-editor.md delete mode 100644 changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 82ff6f33a8..02079c597a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.255.0) + +### Added + +- The repos page's reference repo cards show an entry derived from an accepted request issue read-only: a lock, a `from owner/repo#N` link to the request issue and its credential chip (`🔑 App installation`, `read-only` with the reason, or `unresolved`), with no edit or remove control; `/api/status` `referenceRepos` entries now also carry `derived_from_url`, the request issue's link on the hive's GitHub host (#18139, #18161). +- Dashboard: Settings → Repos has a Reference repos editor for `project.reference_repos`, so a hosted hive's owner can add, change and remove study-only repos without the config file; a save is refused with the loader's or boot's own message (including a write entry with no usable credential, or a `write_token_file` not already set in the file: that path stays operator-only) and otherwise applies to the GitHub proxy without a restart; derived entries are never written to the file (#18136). + +### Fixed + +- The self-host card on `/get-started`, `hub-deployment.md`, `register-self-hosted-hive.md` and the `HIVE_MIRROR_HUB_HIVE_ID` row in `env-vars.md` now show the self-service hive ID as `byo-<login>--<name>`, the double-hyphen form that [#18166](https://github.com/hivecommons/hive/issues/18166) shipped, instead of the earlier single-hyphen form ([#18147](https://github.com/hivecommons/hive/issues/18147)). + ## 2026-11-14 (v5.254.0) ### Added diff --git a/changelog.d/added-18161-repos-card-shows-derived-reference-repos.md b/changelog.d/added-18161-repos-card-shows-derived-reference-repos.md deleted file mode 100644 index fc7a623376..0000000000 --- a/changelog.d/added-18161-repos-card-shows-derived-reference-repos.md +++ /dev/null @@ -1 +0,0 @@ -- The repos page's reference repo cards show an entry derived from an accepted request issue read-only: a lock, a `from owner/repo#N` link to the request issue and its credential chip (`🔑 App installation`, `read-only` with the reason, or `unresolved`), with no edit or remove control; `/api/status` `referenceRepos` entries now also carry `derived_from_url`, the request issue's link on the hive's GitHub host (#18139, #18161). diff --git a/changelog.d/added-reference-repos-editor.md b/changelog.d/added-reference-repos-editor.md deleted file mode 100644 index 60fe6890e3..0000000000 --- a/changelog.d/added-reference-repos-editor.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: Settings → Repos has a Reference repos editor for `project.reference_repos`, so a hosted hive's owner can add, change and remove study-only repos without the config file; a save is refused with the loader's or boot's own message (including a write entry with no usable credential, or a `write_token_file` not already set in the file: that path stays operator-only) and otherwise applies to the GitHub proxy without a restart; derived entries are never written to the file (#18136). diff --git a/changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md b/changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md deleted file mode 100644 index 5720139ec8..0000000000 --- a/changelog.d/fixed-18166-selfservice-docs-double-hyphen-id.md +++ /dev/null @@ -1 +0,0 @@ -- The self-host card on `/get-started`, `hub-deployment.md`, `register-self-hosted-hive.md` and the `HIVE_MIRROR_HUB_HIVE_ID` row in `env-vars.md` now show the self-service hive ID as `byo-<login>--<name>`, the double-hyphen form that [#18166](https://github.com/hivecommons/hive/issues/18166) shipped, instead of the earlier single-hyphen form ([#18147](https://github.com/hivecommons/hive/issues/18147)). From 432289c2bc0906b0c79930f2df0d6af0fbeecf8d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:27:53 +0000 Subject: [PATCH 215/223] fix: log missing-head-SHA verdict refusal at INFO (#18181) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-18180-missing-head-sha-verdict-log-level.md | 1 + src/pkg/github/review_verdict_relay.go | 9 +++++---- src/pkg/github/review_verdict_relay_test.go | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md diff --git a/changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md b/changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md new file mode 100644 index 0000000000..ac8035be35 --- /dev/null +++ b/changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md @@ -0,0 +1 @@ +- Review verdicts discarded for lacking a head SHA and a dispatch are now logged at INFO instead of WARN, so they no longer trigger repeated-error self-reports. diff --git a/src/pkg/github/review_verdict_relay.go b/src/pkg/github/review_verdict_relay.go index 75996c6a04..fe0ebd3fde 100644 --- a/src/pkg/github/review_verdict_relay.go +++ b/src/pkg/github/review_verdict_relay.go @@ -385,12 +385,13 @@ func maintainerCallQuestion(body string) string { return "" } -// logVerdictDispatchRefusal reports a discarded verdict. A self-approval -// refusal is the guard working as designed, not a runtime fault, so it is -// logged at INFO to keep it out of the fleet's repeated-error reports. +// logVerdictDispatchRefusal reports a discarded verdict. A self-approval or +// missing-head-SHA refusal is the guard working as designed, not a runtime +// fault, so it is logged at INFO to keep it out of the fleet's +// repeated-error reports. func (c *Client) logVerdictDispatchRefusal(ctx context.Context, req ReviewRequest, report review.PerspectiveReport, reason string) { level := slog.LevelWarn - if reason == "author_self_approval" { + if reason == "author_self_approval" || reason == "missing_head_sha_without_dispatch" { level = slog.LevelInfo } c.logger.Log(ctx, level, "review-request watcher: verdict does not match a review dispatch, discarded", diff --git a/src/pkg/github/review_verdict_relay_test.go b/src/pkg/github/review_verdict_relay_test.go index 8f4e719aef..87ff5458e1 100644 --- a/src/pkg/github/review_verdict_relay_test.go +++ b/src/pkg/github/review_verdict_relay_test.go @@ -854,6 +854,7 @@ func TestLogVerdictDispatchRefusalLevel(t *testing.T) { wantLevel string }{ {"author_self_approval", "level=INFO"}, + {"missing_head_sha_without_dispatch", "level=INFO"}, {"non_reviewer_without_dispatch", "level=WARN"}, {"dispatch_state_unavailable", "level=WARN"}, } From c47b70da9a7e4d3dfd09d43c1bcfb757012fbbca Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 04:42:50 +0000 Subject: [PATCH 216/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.256.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.256.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ .../added-18161-derived-reference-repos-settings.md | 1 - .../security-18184-proxy-uid-map-runtime-agents.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-18161-derived-reference-repos-settings.md delete mode 100644 changelog.d/security-18184-proxy-uid-map-runtime-agents.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 02079c597a..8b49514042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.256.0) + +### Added + +- Dashboard: Settings → Repos → Reference repos shows entries derived from an accepted request issue as read-only rows with a lock, a `from owner/repo#N` link to the issue, a credential-state chip (`write-tier`, `read-only`, `unresolved`) and the reason; `PUT /api/config/reference-repos` answers 409 to any attempt to edit, remove or replace a derived entry, naming the issue to change instead, while hand-written entries stay editable (#18161, part of #18139). + +### Security + +- The egress proxy now reloads `uid-map.json` when a UID or agent name is not in its copy, so an agent added at runtime is identified from its own socket and protected by the proxy-UID name-claim check ([#18133](https://github.com/hivecommons/hive/issues/18133)) without restarting the hive ([#18184](https://github.com/hivecommons/hive/issues/18184)). + ## 2026-11-14 (v5.255.0) ### Added diff --git a/changelog.d/added-18161-derived-reference-repos-settings.md b/changelog.d/added-18161-derived-reference-repos-settings.md deleted file mode 100644 index 2cd5758a6b..0000000000 --- a/changelog.d/added-18161-derived-reference-repos-settings.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: Settings → Repos → Reference repos shows entries derived from an accepted request issue as read-only rows with a lock, a `from owner/repo#N` link to the issue, a credential-state chip (`write-tier`, `read-only`, `unresolved`) and the reason; `PUT /api/config/reference-repos` answers 409 to any attempt to edit, remove or replace a derived entry, naming the issue to change instead, while hand-written entries stay editable (#18161, part of #18139). diff --git a/changelog.d/security-18184-proxy-uid-map-runtime-agents.md b/changelog.d/security-18184-proxy-uid-map-runtime-agents.md deleted file mode 100644 index effd5ec377..0000000000 --- a/changelog.d/security-18184-proxy-uid-map-runtime-agents.md +++ /dev/null @@ -1 +0,0 @@ -- The egress proxy now reloads `uid-map.json` when a UID or agent name is not in its copy, so an agent added at runtime is identified from its own socket and protected by the proxy-UID name-claim check ([#18133](https://github.com/hivecommons/hive/issues/18133)) without restarting the hive ([#18184](https://github.com/hivecommons/hive/issues/18184)). From 891fd704e0936285cdb35471c9167514a3ca1753 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Sat, 14 Nov 2026 05:14:00 +0000 Subject: [PATCH 217/223] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.256.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.256.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ .../fixed-18180-missing-head-sha-verdict-log-level.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b49514042..115f44df4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-14 (v5.256.1) + +### Fixed + +- Review verdicts discarded for lacking a head SHA and a dispatch are now logged at INFO instead of WARN, so they no longer trigger repeated-error self-reports. + ## 2026-11-14 (v5.256.0) ### Added diff --git a/changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md b/changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md deleted file mode 100644 index ac8035be35..0000000000 --- a/changelog.d/fixed-18180-missing-head-sha-verdict-log-level.md +++ /dev/null @@ -1 +0,0 @@ -- Review verdicts discarded for lacking a head SHA and a dispatch are now logged at INFO instead of WARN, so they no longer trigger repeated-error self-reports. From e053f10b2117207cf0d1af102a2734dd4726fd07 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Sat, 14 Nov 2026 00:37:30 -0500 Subject: [PATCH 218/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20claude?= =?UTF-8?q?=202.1.323=20->=202.1.325=20(#18189)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=2.1.325 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 27866c4f76..c0460b6f7d 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -342,7 +342,7 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_V echo "WARN: OpenAI Codex CLI install failed — skipping" # --- Layer 7: Claude Code (most frequent updates) --- -ARG CLAUDE_CODE_VERSION=2.1.323 +ARG CLAUDE_CODE_VERSION=2.1.325 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during the # build) skips claude-code's install.cjs, which is what LINKS the platform # native binary into the package's bin/. The binary itself ships in the diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 9013899341..5cd2066e3a 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -239,7 +239,7 @@ RUN mkdir -p /etc/apt/keyrings \ # npm cache is cleaned in the same layer it's populated (issue #2613, # Observation 2 "also minor") so the cache doesn't linger in this layer's # diff, mirroring the cleanup already done for the bobshell install below. -ARG CLAUDE_CODE_VERSION=2.1.323 +ARG CLAUDE_CODE_VERSION=2.1.325 ARG COPILOT_VERSION=1.0.88 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during # the build) skips claude-code's install.cjs, which is what LINKS the platform From 2dae878742bc693fe510cd55ce6e1856d58e7411 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Sat, 14 Nov 2026 00:47:05 -0500 Subject: [PATCH 219/223] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20codex?= =?UTF-8?q?=200.168.0=20->=200.169.0=20(#18190)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=0.169.0 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index c0460b6f7d..06f2ed00ce 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -324,7 +324,7 @@ RUN timeout 120s npm install -g --ignore-scripts --no-fund --no-audit \ # dashboard's codex model discovery (`codex app-server` model/list — pure # local stdio, no network, unaffected by the proxy CA) falls back to the # static list, and non-codex backends are unaffected. -ARG CODEX_VERSION=0.168.0 +ARG CODEX_VERSION=0.169.0 ARG CLINE_VERSION=3.0.74 # npm can briefly advertise AWS SDK versions before all registry replicas serve # them. Wait at most three minutes between attempts and revalidate metadata. diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 5cd2066e3a..cd9deb5917 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -262,7 +262,7 @@ ARG KILO_CLI_VERSION=7.5.6 # Same ARG name and value as src/Dockerfile so bin/test_backend_smoke.sh can # assert the two pins agree — this line was previously an inline hardcode that # could (and did) drift from the main image's CODEX_VERSION unnoticed. -ARG CODEX_VERSION=0.168.0 +ARG CODEX_VERSION=0.169.0 RUN npm install -g --ignore-scripts --no-fund --no-audit \ @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ @github/copilot@${COPILOT_VERSION} \ From 673523d6d439f0d87919c79d5ad1ddb93ab261bc Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Sat, 14 Nov 2026 07:49:54 -0500 Subject: [PATCH 220/223] =?UTF-8?q?=F0=9F=8C=B1=20docs:=20re-sync=20a=20fi?= =?UTF-8?q?le:line=20citation=20after=20dashboard=5Fconfig.go=20gained=20t?= =?UTF-8?q?he=20v5=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v5 top-up (#18193) placed v5's SubscriptionGaugeHeader and Sessions fields into v6's split dashboard_config.go, which moved the line that src/docs/labels-and-control-signals/reference.md cites. Re-synced with src/scripts/check-docs-citations.py --fix. Signed-off-by: Andy Anderson <andy@clubanderson.com> --- src/docs/labels-and-control-signals/reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/labels-and-control-signals/reference.md b/src/docs/labels-and-control-signals/reference.md index 3c0da34ccc..187ca1b74e 100644 --- a/src/docs/labels-and-control-signals/reference.md +++ b/src/docs/labels-and-control-signals/reference.md @@ -921,7 +921,7 @@ Queue label for human/owner merge action; the sweep also requires its normal aut **Release or override.** Remove label; head changes can de-queue -**Configuration and implementation.** `governor.labels.automerge`, default `lgtm` (`src/pkg/config/governor_config.go:329-337`, where `LabelsConfig` appears; `src/pkg/config/dashboard_config.go:233`, where `DefaultAutoMergeLabel` appears; `src/pkg/github/automerge/automerge_sweep_loop.go:13-82`) +**Configuration and implementation.** `governor.labels.automerge`, default `lgtm` (`src/pkg/config/governor_config.go:329-337`, where `LabelsConfig` appears; `src/pkg/config/dashboard_config.go:246`, where `DefaultAutoMergeLabel` appears; `src/pkg/github/automerge/automerge_sweep_loop.go:13-82`) #### `reviewer-passed` From e962cc181e53408ba88b0599ad042d27fabbdc26 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Sat, 14 Nov 2026 07:52:11 -0500 Subject: [PATCH 221/223] changelog: fragment for #18193 Signed-off-by: Andy Anderson <andy@clubanderson.com> --- changelog.d/changed-18193.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/changed-18193.md diff --git a/changelog.d/changed-18193.md b/changelog.d/changed-18193.md new file mode 100644 index 0000000000..b535a1d877 --- /dev/null +++ b/changelog.d/changed-18193.md @@ -0,0 +1 @@ +- chore: top up v6 from v5 (2026-11-14) (#18193) From 6d4338298e22f2cad618b374ce99ea77bc3d428a Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Sat, 14 Nov 2026 08:17:52 -0500 Subject: [PATCH 222/223] =?UTF-8?q?=F0=9F=8C=B1=20fix:=20make=20v6-only=20?= =?UTF-8?q?tests=20pass=20on=20the=20merged=20v5=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - agent-cards.js: drop the second terminalCopyHintHtml call the v5 card layout carried; v6 already renders the hint next to the credential-bridge hint, and TestTerminalCopyHints pins the call-site count at two. - navbar_agent_tiles_test.go: stub agentOutOfQuotaShown, which the merged agentCollapsedAuthInfo (#18111) now consults, like the other card helpers the replica-marker harness stubs. - adminmcp: add the reference_repos settings_read section for v5's GET /api/config/reference-repos (#18136) so TestSettingsReadCoversEveryConfigRoute holds, and list it in docs/admin-mcp.md. - heartbeat_mirror.go: classify v6's Runs and UpgradeDrainUntil heartbeat fields as mirrored so TestHeartbeatMirrorFieldPolicyInventory holds. - static/index.html regenerated. Signed-off-by: Andy Anderson <andy@clubanderson.com> --- src/docs/admin-mcp.md | 3 ++- src/pkg/adminmcp/tools_settings.go | 1 + src/pkg/dashboard/navbar_agent_tiles_test.go | 3 +++ src/pkg/dashboard/static-src/js/agent-cards.js | 1 - src/pkg/dashboard/static/index.html | 1 - src/pkg/hub/spoke/heartbeat_mirror.go | 2 ++ 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/docs/admin-mcp.md b/src/docs/admin-mcp.md index 8ed78d7530..d831c9b777 100644 --- a/src/docs/admin-mcp.md +++ b/src/docs/admin-mcp.md @@ -481,7 +481,8 @@ The sections: `question_autoclose`, `question_autoclose_schedule`, `fleet_report`, `work_source`, `project_observability`, `general_advanced`, `cadence_scope`, `threshold_scaling`, `backup`, `bob`, `inference_auth`, - `gateways`, `backends`, `variables`, `write_surface`, `stat_sources`. + `gateways`, `backends`, `variables`, `write_surface`, `stat_sources`, + `reference_repos`. - **Knowledge:** `knowledge_pipeline`, `knowledge_connectors`, `knowledge_connectors_status`, `knowledge_agent_scopes`, `knowledge_public`. - **Panels the tabs load:** `upstream_watch`, `linear_agent`, diff --git a/src/pkg/adminmcp/tools_settings.go b/src/pkg/adminmcp/tools_settings.go index 89ee2babdd..02f32767d8 100644 --- a/src/pkg/adminmcp/tools_settings.go +++ b/src/pkg/adminmcp/tools_settings.go @@ -61,6 +61,7 @@ var settingsSections = []SettingsSection{ {"backends", "/api/config/backends", "Agent backends and their models."}, {"variables", "/api/config/variables", "Variable definitions (names, types and scopes)."}, {"write_surface", "/api/config/write-surface", "Write-surface allowlist."}, + {"reference_repos", "/api/config/reference-repos", "Reference repositories (project.reference_repos): repos the hive may study without managing, each with its write opt-in and token file path (never the token); derived entries name their request issue and credential state."}, {"stat_sources", "/api/config/stat-sources", "Stat sources."}, {"knowledge_pipeline", "/api/config/knowledge/pipeline", "Knowledge pipeline: engine, curator, primer, code maps and bead synthesizer."}, {"knowledge_connectors", "/api/config/knowledge/connectors", "Knowledge connectors (credentials are named, never returned)."}, diff --git a/src/pkg/dashboard/navbar_agent_tiles_test.go b/src/pkg/dashboard/navbar_agent_tiles_test.go index f6b1fa291c..4c76d9ce68 100644 --- a/src/pkg/dashboard/navbar_agent_tiles_test.go +++ b/src/pkg/dashboard/navbar_agent_tiles_test.go @@ -195,6 +195,9 @@ func TestAgentTilesMarkReplicas(t *testing.T) { script := `const assert = require('node:assert/strict'); function agentIsDisabled(a) { return !!a && a.enabled === false; } function _agentNeedsLogin() { return false; } +// The merged agentCollapsedAuthInfo also consults the card's out-of-quota +// predicate (#18111); quota is out of scope for the replica marker. +function agentOutOfQuotaShown() { return false; } function formatDashboardDateTime() { return 'local time'; } ` + jsFunc(t, html, "esc") + ` ` + html[start:start+end] + ` diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index 56426517a5..98c7df23ee 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -1120,7 +1120,6 @@ <span class="working-indicator"></span> <div class="agent-name">${agentOrderHandle}<span class="dot ${dotState}" data-status="${statusAttr(dotState)}"${isOff ? ` title="${esc(agentOffTitle(a))}"` : ''}></span>${a.emoji ? a.emoji + ' ' : ''}${esc(a.displayName || a.name)}${needsLogin ? ' <span title="' + esc(needsLoginBadgeTitle(a)) + '">\uD83D\uDD11</span>' : ''}${a.sandboxed ? ' ' + statusBadgeHTML('ready', 'sandboxed', 'Sandboxed runner active', 'done') : ''}${disabledBadge}${continuousBadge}${replicaBadge} ${agentPowerSwitchHtml(a)} ${isDisabled ? disabledEnableBtn : ''}${toggleBtn}${neverScheduledChipHtml(a, configTarget)} <button class="compact-toggle" data-action="toggleAgentCompact" data-arg0="${esc(a.name)}" data-stop="1" title="Toggle compact/expanded view for this agent card">${isAgentCompact(a.name) ? '▷ expand' : '▽ compact'}</button> ${sessionActions} <button class="hv-btn btn-secondary btn-sm" data-action="showAdvisorRecords" data-arg0="${esc(a.name)}" data-stop="1" title="Advisor reviews of this agent's finished turns — severity and text, newest first">🗣 advisor</button> <button class="config-gear" data-action="openConfigDialog" data-config-type="agent" data-agent="${esc(configTarget)}" title="Configure ${esc(a.replicaBase ? (a.replicaBase + ' (base for ' + a.name + ')') : (a.displayName || a.name))}">⚙️</button></div> ${outOfQuota ? agentOutOfQuotaNoteHtml(a) : ''} - ${terminalCopyHintHtml(needsLogin, agentSessionUnavailable(a), a.name, a.loginURL, a.backendAuthStatus, a.backendAuthLastError, a.statusEvidence, a.inferenceLastError)} <div class="agent-state ${isPoweredOff ? 'disabled' : isOnDemand ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy}">${agentStateText(a)}${pauseInfoIcon(a)}${(() => { if (a.structuredStatus === 'BLOCKED') return statusBadgeHTML('blocked', 'blocked', a.statusEvidence, 'blocked'); if (a.structuredStatus === 'NEEDS_CONTEXT') return statusBadgeHTML('needs_context', 'needs context', a.statusEvidence, 'needs-context'); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 01b2299182..7a46597989 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -32540,7 +32540,6 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> <span class="working-indicator"></span> <div class="agent-name">${agentOrderHandle}<span class="dot ${dotState}" data-status="${statusAttr(dotState)}"${isOff ? ` title="${esc(agentOffTitle(a))}"` : ''}></span>${a.emoji ? a.emoji + ' ' : ''}${esc(a.displayName || a.name)}${needsLogin ? ' <span title="' + esc(needsLoginBadgeTitle(a)) + '">\uD83D\uDD11</span>' : ''}${a.sandboxed ? ' ' + statusBadgeHTML('ready', 'sandboxed', 'Sandboxed runner active', 'done') : ''}${disabledBadge}${continuousBadge}${replicaBadge} ${agentPowerSwitchHtml(a)} ${isDisabled ? disabledEnableBtn : ''}${toggleBtn}${neverScheduledChipHtml(a, configTarget)} <button class="compact-toggle" data-action="toggleAgentCompact" data-arg0="${esc(a.name)}" data-stop="1" title="Toggle compact/expanded view for this agent card">${isAgentCompact(a.name) ? '▷ expand' : '▽ compact'}</button> ${sessionActions} <button class="hv-btn btn-secondary btn-sm" data-action="showAdvisorRecords" data-arg0="${esc(a.name)}" data-stop="1" title="Advisor reviews of this agent's finished turns — severity and text, newest first">🗣 advisor</button> <button class="config-gear" data-action="openConfigDialog" data-config-type="agent" data-agent="${esc(configTarget)}" title="Configure ${esc(a.replicaBase ? (a.replicaBase + ' (base for ' + a.name + ')') : (a.displayName || a.name))}">⚙️</button></div> ${outOfQuota ? agentOutOfQuotaNoteHtml(a) : ''} - ${terminalCopyHintHtml(needsLogin, agentSessionUnavailable(a), a.name, a.loginURL, a.backendAuthStatus, a.backendAuthLastError, a.statusEvidence, a.inferenceLastError)} <div class="agent-state ${isPoweredOff ? 'disabled' : isOnDemand ? 'on-demand' : isPaused ? 'paused' : isOff ? 'off' : a.busy}">${agentStateText(a)}${pauseInfoIcon(a)}${(() => { if (a.structuredStatus === 'BLOCKED') return statusBadgeHTML('blocked', 'blocked', a.statusEvidence, 'blocked'); if (a.structuredStatus === 'NEEDS_CONTEXT') return statusBadgeHTML('needs_context', 'needs context', a.statusEvidence, 'needs-context'); diff --git a/src/pkg/hub/spoke/heartbeat_mirror.go b/src/pkg/hub/spoke/heartbeat_mirror.go index aa8f03b5b6..074ce00f9d 100644 --- a/src/pkg/hub/spoke/heartbeat_mirror.go +++ b/src/pkg/hub/spoke/heartbeat_mirror.go @@ -136,6 +136,7 @@ var heartbeatMirrorFieldPolicy = map[string]mirrorFieldPolicy{ "ACMMLevel": mirrorFieldMirrored, "Agents": mirrorFieldMirrored, "Governor": mirrorFieldMirrored, + "Runs": mirrorFieldMirrored, "Tokens24h": mirrorFieldMirrored, "Contributors": mirrorFieldMirrored, "Leaderboard": mirrorFieldMirrored, @@ -185,6 +186,7 @@ var heartbeatMirrorFieldPolicy = map[string]mirrorFieldPolicy{ "UpgradeError": mirrorFieldMirrored, "UpgradePendingRestart": mirrorFieldMirrored, "UpgradeDrain": mirrorFieldMirrored, + "UpgradeDrainUntil": mirrorFieldMirrored, // Fleet contribution, repo activity and quadrant signals. "PRsMerged90d": mirrorFieldMirrored, "PRsRejected90d": mirrorFieldMirrored, From 7001fd89fbb43c159e2909f8fc8a348f8fa95fac Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Sat, 14 Nov 2026 08:42:54 -0500 Subject: [PATCH 223/223] Third-Party DCO Remediation Commit for hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> On behalf of hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com>, I, Andy Anderson <andy@clubanderson.com>, hereby add my Signed-off-by to this commit: f33758f5ad4ac50a458a7c4a2120547126f5b039 The squash of #18079 dropped the branch commit's Signed-off-by (c15eb728d was signed). See #18203. Signed-off-by: Andy Anderson <andy@clubanderson.com>