From 9b4cdb68d22df50aa3059eb314a66cbe12af2a3e Mon Sep 17 00:00:00 2001 From: convee Date: Sat, 5 Sep 2026 12:10:45 +0800 Subject: [PATCH 1/2] feat: add scalable kubeadm runtime workers --- Makefile | 14 +- README.md | 67 +- control_plane/core.py | 3 +- docs/ARCHITECTURE.md | 14 +- docs/COMPATIBILITY.md | 11 +- docs/CONFIGURATION.md | 2 +- docs/DEPLOYMENT.md | 79 ++- docs/PLATFORM_CONTRACT.md | 18 +- docs/SECURITY_MODEL.md | 6 +- docs/SYSTEM_SPECIFICATIONS.md | 2 +- docs/TROUBLESHOOTING.md | 75 ++- docs/index.html | 35 +- overlays/local-dev/kustomization.yaml | 1 - .../workspace-storage-local-path.yaml | 3 + overlays/local/kustomization.yaml | 4 +- rook/ceph-csi-drivers-values.yaml | 49 ++ rook/cluster-local.yaml | 57 +- scripts/dev-doctor.sh | 70 +- scripts/install-cilium-kubeadm.sh | 6 +- scripts/local-cluster.sh | 612 ++++++++++++++++-- scripts/local-cluster.yaml | 13 +- scripts/quickstart.sh | 26 +- scripts/test.sh | 17 + tests/test_dev_scripts.py | 85 ++- tests/test_documentation.py | 28 +- tests/test_reaper_behavior.py | 8 + tests/test_standalone_contract.py | 4 +- tests/test_storage_modes.py | 83 ++- 28 files changed, 1214 insertions(+), 178 deletions(-) create mode 100644 rook/ceph-csi-drivers-values.yaml diff --git a/Makefile b/Makefile index 0b733fb..6143113 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ # Standalone deployment surface for this repository. -# The only cluster profile is single-VM Lima + kubeadm + Cilium + gVisor. +# The local profile is one Lima + kubeadm cluster: trusted system services on +# the control-plane node, gVisor Runtime workloads on a scalable tainted worker pool. # Image build contexts are deliberately rooted in this repository. SHELL := /usr/bin/env bash SANDBOX_KUBE_CONTEXT ?= sandbox-local @@ -20,7 +21,7 @@ SANDBOX_BUILD_LOG_DIR ?= $(SANDBOX_STATE_DIR)/logs/images DOCKER_BUILD_PROGRESS ?= plain SANDBOX_BOOTSTRAP_STAMP ?= .venv/.sandbox-bootstrap -.PHONY: help bootstrap quickstart acceptance smoke-local e2e-local verify verify-manifests doctor images image-runtime image-file-service image-control-plane image-console test chart-lint chart-render dev-token control-plane-forward console-forward up-local status-local down-local destroy-local +.PHONY: help bootstrap quickstart acceptance smoke-local e2e-local verify verify-manifests doctor images image-runtime image-file-service image-control-plane image-console test chart-lint chart-render dev-token control-plane-forward console-forward up-local status-local scale-workers down-local destroy-local help: @grep -E '^[a-z-]+:.* — ' Makefile | sed 's/:.* — / — /' @@ -49,6 +50,7 @@ smoke-local: ## — prove gVisor isolation, persistence, fail-closed behavior, a e2e-local: ## — run all network, Runtime, storage, restart, and adversarial E2E checks SANDBOX_KUBECONFIG=$(KUBECONFIG) \ SANDBOX_KUBE_CONTEXT=$(SANDBOX_KUBE_CONTEXT) \ + SANDBOX_EXPECT_DEDICATED_RUNTIME_NODE=1 \ PYTHON=$(SANDBOX_PYTHON) bash scripts/run-all-e2e.sh verify: ## — run the complete source, package, Console, and manifest gate with logs @@ -104,8 +106,12 @@ up-local: ## — deploy the standalone Lima/kubeadm environment status-local: ## — show the standalone Lima/kubeadm environment bash scripts/local-cluster.sh status -down-local: ## — stop the Lima VM and retain its disk +scale-workers: ## — resize the Runtime worker pool with WORKERS=N + @test -n "$(WORKERS)" || { echo 'usage: make scale-workers WORKERS=N' >&2; exit 2; } + bash scripts/local-cluster.sh scale-workers "$(WORKERS)" + +down-local: ## — stop the control plane and all Runtime workers; retain disks bash scripts/local-cluster.sh down -destroy-local: ## — delete the Lima VM, .sandbox state, and local images +destroy-local: ## — delete all profile VMs, .sandbox state, and local images bash scripts/local-cluster.sh destroy diff --git a/README.md b/README.md index 5ec12a5..5c09926 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ operation fails; it never falls back to running on the host. │ the tenant is decided by the credential, │ never by anything in the request body ════════════════════════════════▼═════════════════════════════════════════════ - namespace: sandbox-system trusted control plane + node: sandbox-local · namespace: sandbox-system trusted control plane ┌─────────────────────────────────────────────────────────────────────────┐ │ Control Plane (control_plane/) │ │ admission · quotas · tokens · workspace and runtime lifecycle │ @@ -33,7 +33,7 @@ operation fails; it never falls back to running on the host. └──────────────────────────────┬──────────────────────────────────────────┘ │ in-cluster, NetworkPolicy-scoped ════════════════════════════════▼═════════════════════════════════════════════ - namespace: sandbox-workloads default-deny, untrusted workload + nodes: sandbox-local-w1…wN · namespace: sandbox-workloads Runtime worker pool ┌─────────────────────────────────────────────────────────────────────────┐ │ Runtime Pod RuntimeClass: gvisor │ │ runtime/ shell, PTY sessions, SSE streaming │ @@ -50,7 +50,7 @@ operation fails; it never falls back to running on the host. ```bash make bootstrap # create .venv and install SDK + test dependencies -make test # 839 unit and contract tests, no network, no cluster +make test # 844 unit and contract tests, no network, no cluster make verify # complete Python, Console, manifest, Helm, wheel gate make help # every Make target with its one-line description ``` @@ -187,8 +187,11 @@ that order with one command. ## Run the full local cluster -`make up-local` builds a single-node kubeadm Kubernetes cluster inside a Lima VM with -Cilium and gVisor, then deploys the whole platform into it. This is the only local +`make up-local` builds one multi-node kubeadm Kubernetes cluster in Lima. The +`sandbox-local` node hosts trusted system services; the tainted +`sandbox-local-w1…wN` pool hosts gVisor Runtime workloads. Cilium connects the +nodes and enforces the workload policies; CephFS keeps Workspaces mountable from +any active worker. The default is one worker, not a fixed maximum. This is the only local cluster profile the project ships. ### Prerequisites @@ -202,17 +205,29 @@ it first rather than discovering a gap halfway through the VM build. | Python | 3.11 or newer | | Host OS | macOS or Linux | | Host architecture | amd64 or arm64 (`scripts/local-cluster.yaml` pins Ubuntu images for both; gVisor is installed for `x86_64` and `aarch64`) | -| **Available memory** | **6.5 GiB free** for a new profile — the default VM reserves 6 GiB; this is a hard check, not a warning | -| **Free disk** | **25 GiB free** under `$LIMA_HOME` (default `~/.lima`) for a new profile — the 60 GiB VM disk is sparse; this is also a hard check | +| **Available memory** | **10.5 GiB free** for the default new profile — 6 GiB for the control plane plus 4 GiB per worker; this is a hard check, not a warning | +| **Free disk** | **35 GiB free** under `$LIMA_HOME` for the default new profile — the 60 GiB control-plane and 30 GiB-per-worker disks are sparse | | Virtualization | On Linux, a readable and writable `/dev/kvm`. Without it Lima falls back to QEMU TCG software emulation, which boots kubeadm many times slower and is not usable in practice. `make doctor` warns rather than fails on this one. | | Network | Egress to pull the Ubuntu cloud image, Kubernetes apt packages, Cilium, gVisor, Metrics Server, and Rook/Ceph images | Set `SANDBOX_DOCTOR_SKIP_RESOURCES=1` to bypass only the memory and disk checks. The -VM itself uses 4 CPUs, 6 GiB memory, and a 60 GiB disk by default, adjustable through -`SANDBOX_LOCAL_CPUS`, `SANDBOX_LOCAL_MEMORY_GIB`, and `SANDBOX_LOCAL_DISK_GIB`. -When `sandbox-local` already exists, doctor switches to a 2 GiB memory / 5 GiB disk -reuse gate because it is not allocating a second VM; explicit -`SANDBOX_DOCTOR_MIN_*` overrides still win. +control-plane VM uses 4 CPUs, 6 GiB memory, and a 60 GiB disk by default, +adjustable through `SANDBOX_LOCAL_CPUS`, `SANDBOX_LOCAL_MEMORY_GIB`, and +`SANDBOX_LOCAL_DISK_GIB`. The Runtime worker uses 4 CPUs, 4 GiB memory, and a +30 GiB disk. `SANDBOX_LOCAL_WORKER_COUNT` selects the initial pool size; +`SANDBOX_LOCAL_WORKER_CPUS`, `SANDBOX_LOCAL_WORKER_MEMORY_GIB`, and +`SANDBOX_LOCAL_WORKER_DISK_GIB` size each worker. When all requested VMs already +exist, doctor switches to a 2 GiB memory / 5 GiB disk reuse gate; expanding the +pool adds capacity checks per missing worker. Explicit `SANDBOX_DOCTOR_MIN_*` +overrides still win. When VM memory or disk sizes are customized, the resource +gate derives its reservation from those configured sizes rather than retaining +the default-profile threshold. + +Every profile owns a Lima `user-v2` network named `-net`; this is +what gives each VM a distinct, mutually reachable address. Override it with +`SANDBOX_LOCAL_NETWORK`; override its automatically selected unused `/24` with +`SANDBOX_LOCAL_NETWORK_GATEWAY`. An older VM created on Lima's isolated default usernet +is rejected rather than silently reused as a broken multi-node cluster. ### Bring it up @@ -223,7 +238,7 @@ make up-local ``` The first `make up-local` downloads and builds everything listed above, so its -duration is dominated by your network throughput; subsequent runs reuse the VM disk. +duration is dominated by your network throughput; subsequent runs reuse the existing VM disks. It ends by printing: ```text @@ -236,6 +251,20 @@ The kubeconfig is written to `.sandbox/kubeconfig` inside the checkout, not to The Makefile exports `KUBECONFIG` for its own targets, so `make dev-token`, `make status-local`, and the port-forward targets need no manual export. +Resize the Runtime pool without rebuilding the control plane: + +```bash +make scale-workers WORKERS=3 # add/start w2 and w3, install gVisor, load Runtime image +make scale-workers WORKERS=1 # drain and stop w2 and w3; disks are retained +make scale-workers WORKERS=0 # allowed only when no Runtime Pods remain +``` + +Scale-down fails closed if a target node still has an active Runtime Pod. Release +or checkpoint that Runtime first, then retry. Scaling back up reuses stopped worker +disks; the stale Kubernetes Node registration is removed while stopped and kubeadm +registers it again on scale-up. At zero workers, Runtime admission +is paused and fails fast; the trusted storage and Console services remain available. + ### First command ```bash @@ -256,11 +285,11 @@ Workspace `demo`. ### Tear it down ```bash -make down-local # stop the Lima VM; its 60 GiB disk and .sandbox/ remain -make destroy-local # delete the VM with its disk and the generated .sandbox/ state +make down-local # stop the control plane and every worker; VM disks and .sandbox/ remain +make destroy-local # delete the control plane, every worker, and generated state ``` -`down-local` is the right choice between sessions — `up-local` reuses the stopped VM. +`down-local` is the right choice between sessions — `up-local` reuses both stopped VMs. `destroy-local` is irreversible: Workspace files, checkpoints, and the SQLite state inside the VM are gone. It also removes the four fixed-tag project images from the local Docker daemon; shared base layers remain available to Docker's cache. @@ -325,8 +354,8 @@ rather than a deployable overlay; CI checks the others on every push. | Profile | Intended for | Maturity | | --- | --- | --- | -| `overlays/local` | The `make up-local` VM. SQLite state, local-path PVCs, single pinned volume agent. | Reference. Exercises isolation and recovery behavior; its single-node storage and SQLite database are **not** a production durability claim. | -| `overlays/rwo-single-node` | Clusters without RWX storage. Trades volume-agent availability for portability: one RWO claim, one pinned replica. | Reference. | +| `overlays/local` | The `1+N` `make up-local` cluster. SQLite state, CephFS RWX Workspaces, and a scalable Runtime pool. | Reference. Exercises placement, isolation, scaling, and recovery behavior; its single-OSD Ceph and SQLite database are **not** production durability claims. | +| `overlays/rwo-single-node` | Clusters where all Workspace consumers share one RWO-capable Runtime node. Trades volume-agent availability for portability: one RWO claim, one pinned replica. | Reference. | | `overlays/eks` | Amazon EKS. Swaps in an EFS-backed StorageClass, which the operator must install first. | Adapter example — see [Known limitations](#known-limitations) before using it. | | `overlays/external-deps` | Control-plane state and object storage outside the cluster (managed database, S3-compatible store). | Example with placeholders. Nothing references it by default; copy it and fill in your own Secrets. | | `charts/sandbox` | Helm-based installs. `make chart-lint` and `make chart-render` validate it without a cluster. | Independently deployable package. | @@ -373,7 +402,7 @@ the image references before treating this overlay as deployable. **The `sandbox-system` namespace has no NetworkPolicy.** All four shipped NetworkPolicies target `sandbox-workloads`. The Control Plane and Console are also exposed as NodePort services (30080 and 30081) in the base manifests. Both are -acceptable for a single-node local VM and are not acceptable on a shared cluster +acceptable for the dedicated local cluster and are not acceptable on a shared cluster without an ingress and namespace policies in front of them. **The workspace admission gate is a process-local lock.** diff --git a/control_plane/core.py b/control_plane/core.py index cdc5760..07daa5e 100644 --- a/control_plane/core.py +++ b/control_plane/core.py @@ -638,10 +638,11 @@ def object_store_env( ("CHECKPOINT_RETENTION_SECONDS", CHECKPOINT_RETENTION_SECONDS), ("CHECKPOINT_GC_INTERVAL_SECONDS", CHECKPOINT_GC_INTERVAL_SECONDS), ("SANDBOX_MAX_WORKSPACES", MAX_WORKSPACES), - ("SANDBOX_MAX_RUNTIMES", MAX_RUNTIMES), ): if _value <= 0: raise ValueError(f"{_name} must be greater than zero") +if MAX_RUNTIMES < 0: + raise ValueError("SANDBOX_MAX_RUNTIMES must be zero or greater") MAX_CONCURRENT_OBJECT_OPS = int( os.getenv("SANDBOX_MAX_CONCURRENT_OBJECT_OPS", "1") ) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0f86409..9c8dfe1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -112,7 +112,7 @@ or memory snapshots. See [Lifecycle and data](LIFECYCLE_AND_DATA.md). | Profile | Isolation and dependencies | Intended use | | --- | --- | --- | -| Local integration | gVisor, Cilium, SQLite, local-path Workspace PVC, and cluster-local Ceph RGW | Reproducible isolation and E2E validation | +| Local integration | gVisor, Cilium, SQLite, CephFS RWX Workspace PVC, and cluster-local Ceph RGW | Reproducible isolation, scaling, and E2E validation | | Operator production | gVisor nodes, enforcing CNI, PostgreSQL, external RWX and S3-compatible storage | ACK, EKS, or any conforming Kubernetes platform | The same Control Plane/Runtime contract applies across profiles. The local profile proves @@ -150,6 +150,18 @@ provider is selected independently by `SANDBOX_RUNTIME_DRIVER`. The repository is self-contained: build, test, and local-cluster entrypoints resolve paths from this checkout. +The repository-managed local profile uses one kubeadm cluster with a trusted +control-plane node and a scalable pool of tainted gVisor Runtime workers. This is +node isolation inside one Kubernetes trust domain, not a dual-cluster claim. +Workers become schedulable only after node readiness, gVisor installation, and +Runtime image loading. Scale-down first validates the entire target set and stops +no node when any target still owns an active Runtime; it cordons and rechecks all +targets to close the scheduler race, rolling those cordons back on refusal. CephFS RWX makes Workspace +state independent of worker identity; scaling the pool to zero pauses Runtime +admission without removing the trusted system and storage node. Stopped worker +disks are retained, while their ephemeral Kubernetes Node registrations are removed +and recreated on the next scale-up so node-wide DaemonSet rollouts remain healthy. + Several control-plane modules remain larger than their domain boundaries. Future refactors should extract authentication, Workspace, Runtime, object, and admin route groups without changing the OpenAPI contract or authorization behavior. Route, diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 60fa4a4..1a75fa4 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -13,14 +13,21 @@ not supported until a versioned compatibility suite proves otherwise. | Console | Node.js 22 build environment | Locked npm install, i18n, ESLint, TypeScript, and production build | | Kubernetes manifests | Provider-neutral base, local integration, EKS | Kustomize render and contract tests | +The currently validated local integration stack is kubeadm/Kubernetes 1.36, +Cilium 1.19.6, Rook 1.20.6, Ceph 20.2.4, and Ceph-CSI 3.17.1 on Ubuntu Noble. +Rook and Ceph-CSI are separate checksum-pinned Helm charts; the Ceph and Ceph-CSI +images are digest-pinned. Noble's 6.8 kernel uses the CephFS FUSE mounter in this +profile. Treat any change to that tuple as a compatibility change that requires +the full local E2E suite. + ## Compatibility rules - Kubernetes must support RuntimeClass, NetworkPolicy, batch/v1 Jobs/CronJobs, leases, and the storage access mode selected by the overlay. - The `gvisor` RuntimeClass must resolve to a working `runsc` handler on every node eligible for Runtime Pods. An object alone is not proof of isolation. -- Production deployments must replace development SQLite, the single-node Ceph - profile, and local-path storage with the dependencies and backup policies in +- Production deployments must replace development SQLite and the single-OSD Ceph + profile with the dependencies and backup policies in [Production](PRODUCTION.md). - The local profile provides Metrics Server only through its overlay. Production must provide its own Metrics API and trusted kubelet serving certificate path. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index c2d33de..4311193 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -144,7 +144,7 @@ attributed to a person, so: | `SANDBOX_TEMPLATES` | built-in default only | JSON object mapping approved template IDs to image references | | `SANDBOX_IMAGE_REGISTRIES` | empty | Allowed literal image prefixes; empty fails closed for API-managed templates | | `SANDBOX_MAX_WORKSPACES` | `64` | Global workspace admission cap | -| `SANDBOX_MAX_RUNTIMES` | `4` | Global Runtime admission cap | +| `SANDBOX_MAX_RUNTIMES` | `4` | Global Runtime admission cap; `0` pauses new Runtime admission | | `SANDBOX_TTL_SECONDS` | `1800` | Idle Runtime TTL | | `SANDBOX_RUNTIME_HARD_TTL_SECONDS` | `43200` | Absolute Runtime lifetime | | `WORKSPACE_IDLE_TTL_SECONDS` | `21600` | Workspace data idle TTL | diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 9d3292a..94f482f 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -34,10 +34,11 @@ Lima provides the Linux VM in the standalone integration environment; kubeadm provides Kubernetes inside those VMs. Sandbox Platform itself talks to Kubernetes and does not depend on the Lima API. -The standalone integration entrypoint creates one dedicated Lima VM, bootstraps a -kubeadm cluster, installs checksum-pinned Cilium and gVisor, loads locally built -images plus the pinned Metrics Server image into containerd, and applies a -self-contained development profile: +The standalone integration entrypoint creates one control-plane VM plus a +configurable pool of Runtime-worker VMs in one kubeadm cluster. The control-plane +node hosts trusted system services; tainted workers host Runtime workloads. It +installs checksum-pinned Cilium cluster-wide and gVisor only on workers, loads each image only onto nodes that +need it, and applies a self-contained development profile: ```bash make quickstart @@ -52,12 +53,15 @@ proof. It records phase timing and outcome in `.sandbox/quickstart-summary.json`. Use `make up-local` directly when the Python environment is already prepared and only the deployment needs updating. -For a new profile, `make doctor` fails when less than 6.5 GiB of memory or 25 GiB -of disk is free. When the dedicated `sandbox-local` VM already exists, it uses a -2 GiB memory / 5 GiB disk reuse gate instead. It +For the default one-worker profile, `make doctor` fails when less than 10.5 GiB +of memory or 35 GiB of disk is free. Each additional missing worker adds its +configured capacity to the gate. When all requested nodes already exist, it uses +a 2 GiB memory / 5 GiB disk reuse gate instead. It warns when `/dev/kvm` is absent on Linux (Lima then falls back to QEMU software emulation, which is far slower). Set `SANDBOX_DOCTOR_SKIP_RESOURCES=1` to skip the -resource gate. +resource gate. Overrides to the control-plane or worker VM memory/disk sizes also +change the calculated new-profile and expansion thresholds; the defaults remain +10.5 GiB memory and 35 GiB disk. The cluster-admin kubeconfig is written to `.sandbox/kubeconfig`, never to `~/.kube/config`. Make targets that talk to the cluster (`dev-token`, @@ -69,12 +73,36 @@ export KUBECONFIG="$(scripts/local-cluster.sh kubeconfig)" kubectl get nodes ``` -Required host tools are Docker, Lima, kubectl, Helm, and OpenSSL. The default VM is -`sandbox-local`; `down-local` stops it without deleting its disk. Override the -name or state directory with `SANDBOX_LOCAL_VM` and `SANDBOX_STATE_DIR`. +Required host tools are Docker, Lima, kubectl, Helm, and OpenSSL. The default nodes +are `sandbox-local` and workers `sandbox-local-w1…wN`; `down-local` stops all +without deleting disks. Override the control-plane name with `SANDBOX_LOCAL_VM`, +the worker prefix/count with `SANDBOX_LOCAL_WORKER_PREFIX` and +`SANDBOX_LOCAL_WORKER_COUNT`, the profile-owned Lima user-v2 network with +`SANDBOX_LOCAL_NETWORK`, its automatically selected `/24` with +`SANDBOX_LOCAL_NETWORK_GATEWAY`, and state with `SANDBOX_STATE_DIR`. Each VM receives a +distinct address on that shared network; an existing VM on Lima's isolated +default usernet is rejected because it cannot form a multi-node cluster. -Remove everything the profile created on the host, that is the Lima VM and its -disk, the `.sandbox` state directory including the kubeconfig, and the four locally +Resize the active Runtime pool without rebuilding the control plane: + +```bash +make scale-workers WORKERS=3 +make scale-workers WORKERS=1 +make scale-workers WORKERS=0 +``` + +The command makes a new worker schedulable only after it is Ready, gVisor is +installed, and the Runtime image is loaded. Before scale-down it checks every +target node, cordons the complete target set, and checks again to close the +scheduler race. If either check finds a Runtime Pod, no worker is stopped and any +cordon added by the command is rolled back. At zero +workers the Control Plane sets Runtime admission capacity to zero, while Console +and durable Workspace services remain available. A successful scale-down preserves +the Lima disk but removes the stopped machine's Kubernetes Node registration and +orphaned DaemonSet Pods; kubeadm registers the retained worker again on scale-up. + +Remove everything the profile created on the host, that is the control-plane VM and every worker VM with their +disks, the `.sandbox` state directory including the kubeconfig, and the four locally built images, with: ```bash @@ -91,7 +119,10 @@ SANDBOX_STATE_DIR="$PWD/.sandbox-test" \ SANDBOX_LOCAL_API_PORT=28448 \ SANDBOX_LOCAL_CONTROL_PLANE_PORT=28080 \ SANDBOX_LOCAL_CPUS=2 \ -SANDBOX_LOCAL_MEMORY_GIB=4 \ +SANDBOX_LOCAL_MEMORY_GIB=6 \ +SANDBOX_LOCAL_WORKER_COUNT=2 \ +SANDBOX_LOCAL_WORKER_CPUS=4 \ +SANDBOX_LOCAL_WORKER_MEMORY_GIB=4 \ make up-local ``` @@ -99,14 +130,22 @@ Port overrides belong to the VM at creation time. When reusing an existing VM, the script rejects mismatched values instead of silently writing a broken kubeconfig. -This profile proves the Lima/kubeadm/gVisor path, but uses single-node local-path -storage, SQLite, and an in-cluster Ceph RGW development profile backed by local disk. +This profile proves the Lima/kubeadm/gVisor path, dedicated Runtime-node placement, +and worker scaling. CephFS provides RWX Workspaces to every worker, but SQLite and +the single-OSD Ceph development profile remain backed by the control-plane node's local disk. Data survives Pod restarts but not deletion of the VM disk. It is not a production durability or HA claim. -`scripts/local-cluster.sh` installs the Rook chart, applies -`rook/cluster-local.yaml`, waits for `CephCluster` and `CephObjectStore` readiness, -copies the generated `sandbox-runtime` user into `object-store-credentials`, and then +`scripts/local-cluster.sh` installs the checksum-pinned Rook operator chart and the +separate checksum-pinned `ceph-csi-drivers` chart, then applies +`rook/cluster-local.yaml` and waits for the CSI controllers/node plugins, +`CephCluster`, `CephFilesystem`, and `CephObjectStore` readiness. The local stack +pins Ceph 20.2.4 and Ceph-CSI 3.17.1 by digest. Its `sandbox-rwx` StorageClass uses +the userspace FUSE mounter so Ubuntu Noble's 6.8 kernel does not need to consume +the newer AES256K CephX key format directly. The installer recreates an obsolete +StorageClass only when no PV or PVC references it; otherwise it fails with an +explicit migration message rather than touching Workspace data. It then copies +the generated `sandbox-runtime` user into `object-store-credentials`, and then runs the versioned bucket-initialization Job. The endpoint is `http://rook-ceph-rgw-object-store.rook-ceph.svc.cluster.local:80`; the buckets are `user-uploads`, `agent-data`, and `sandbox-workspaces`. @@ -120,7 +159,7 @@ images pinned to the exact signed GHCR digests. Supply the | Environment | Kube context convention | Configuration source | Storage/database | | --- | --- | --- | --- | -| Local | `sandbox-local` | `overlays/local`, generated local Secrets | SQLite, in-cluster Ceph RGW, local-path RWO Workspace storage | +| Local | `sandbox-local` | `overlays/local`, generated local Secrets | SQLite, in-cluster Ceph RGW, CephFS RWX Workspace storage | | Development | `sandbox-dev` | Base plus operator GitOps values | PostgreSQL, external object store, RWX CSI | | Staging | `sandbox-staging` | Base plus operator GitOps values | Production-equivalent managed dependencies | | Production | `sandbox-prod` | Base plus reviewed provider adapter | Durable managed dependencies and backup | diff --git a/docs/PLATFORM_CONTRACT.md b/docs/PLATFORM_CONTRACT.md index c255452..480e1b6 100644 --- a/docs/PLATFORM_CONTRACT.md +++ b/docs/PLATFORM_CONTRACT.md @@ -10,10 +10,10 @@ selected implementation and passes the repository acceptance suite. | Runtime isolation | Kubernetes `RuntimeClass` | gVisor `runsc` | Install and verify the configured handler on every eligible Runtime node | | Control state | `SANDBOX_STORE_BACKEND` contract | SQLite | PostgreSQL with durable storage, backup, and schema-compatible rollback | | Object storage | S3-compatible endpoint and `OBJECT_STORE_*` credentials | Cluster-local Ceph RGW development profile | TLS-enabled S3-compatible service with versioning, retention, least-privilege credentials, and tested restore behavior | -| Workspace files | Kubernetes PVC and CSI `StorageClass` | Shared RWO local-path claim | Shared RWX claim, verified capacity, snapshots/backup, and restore testing | +| Workspace files | Kubernetes PVC and CSI `StorageClass` | CephFS RWX claim | Shared RWX claim, verified capacity, snapshots/backup, and restore testing | | High-performance disk | CSI `StorageClass` selected by overlay | Not claimed | Provider-specific class that satisfies the selected access mode and failure-domain contract | | Secrets | Kubernetes `Secret` references | Random local bootstrap values | External secret manager and audited rotation | -| Elastic Runtime capacity | Kubernetes node labels, taints, and `RuntimeClass` | A pre-created local worker | A node-pool controller that verifies Node Ready, CNI, runtime handler, and workload schedulability | +| Elastic Runtime capacity | Kubernetes node labels, taints, and `RuntimeClass` | `sandbox-local-w1…wN`, managed by `make scale-workers` | A node-pool controller that verifies Node Ready, CNI, runtime handler, and workload schedulability | ## Stable and elastic nodes @@ -28,10 +28,16 @@ The Helm values `scheduling.system.*` and `runtime.*` are the public integration surface. An infrastructure project may populate those ordinary values, but the Sandbox does not call an infrastructure repository or depend on its environment variables. Both values default to empty so the standalone chart works on an -unlabelled single-node Kubernetes cluster. A production node-pool adapter must set +unlabelled Kubernetes cluster. The local installer supplies the label and taint +for every Runtime worker. A production node-pool adapter must set both sides of the scheduling contract. Scaling a Runtime pool to zero is permitted only after active Runtime Pods have drained; workspace durability comes from the -PVC/object-store contract, not from a worker's local filesystem. +PVC/object-store contract, not from a worker's local filesystem. The local adapter +preflights all scale-down targets, cordons and rechecks them as a set before +stopping any of them, updates Runtime +admission and namespace quota with the worker count, and keeps a new node cordoned +until its Runtime image is available. At zero, new Runtime admission fails fast; +system and Workspace access remain online. A worker is usable only after the node is Ready, the CNI is healthy, the selected `RuntimeClass` handler works, and a probe Pod can be scheduled and become Ready. @@ -59,8 +65,8 @@ exact target only through `SANDBOX_KUBE_CONTEXT`. ## Storage constraints The portable topology uses one shared claim and mounts a workspace-specific -`subPath` into each Runtime. This works with RWO on the single-node local profile -and requires RWX for multi-node production. Requested PVC capacity is not proof of +`subPath` into each Runtime. The multi-node local profile supplies CephFS RWX; +multi-node production likewise requires RWX. Requested PVC capacity is not proof of hard quota enforcement. Optional per-workspace PVC mode is not portable by StorageClass substitution alone: diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md index c618509..971bb96 100644 --- a/docs/SECURITY_MODEL.md +++ b/docs/SECURITY_MODEL.md @@ -53,8 +53,10 @@ unbounded Runtime lifetime, and accidental host fallback when Sandbox is unavail - The operator console is not a secret store and must sit behind an authenticated administrative ingress. - The `shared` storage mode weakens storage isolation and quota semantics. -- The local overlay deliberately uses SQLite and single-node storage; it validates - gVisor isolation but not production durability or high availability. +- The local overlay deliberately uses SQLite and single-OSD Ceph. Its scalable, + tainted Runtime worker pool validates placement and gVisor isolation, but + the two nodes remain in one Kubernetes trust domain and do not prove production + durability or high availability. - Side-channel resistance, malicious kernel exploits beyond gVisor's guarantees, and public anonymous multi-tenancy are not claimed. diff --git a/docs/SYSTEM_SPECIFICATIONS.md b/docs/SYSTEM_SPECIFICATIONS.md index 75eb3b4..63ca483 100644 --- a/docs/SYSTEM_SPECIFICATIONS.md +++ b/docs/SYSTEM_SPECIFICATIONS.md @@ -12,7 +12,7 @@ operator-managed template. | Isolation class | `gvisor` | `SANDBOX_RUNTIME_CLASS` | | Runtime lifetime | 1,800 seconds idle; 43,200 seconds absolute | Control Plane environment | | Scoped access token | 900 seconds | `ACCESS_TOKEN_TTL_SECONDS` | -| Concurrent runtimes | 4 per Control Plane deployment | `SANDBOX_MAX_RUNTIMES` | +| Concurrent runtimes | 4 by default; local profile scales at 4 per active Runtime worker | `SANDBOX_MAX_RUNTIMES` (`0` pauses admission) | | Workspaces | 64 per Control Plane deployment | `SANDBOX_MAX_WORKSPACES` | | Workspace request | 1 GiB | `SANDBOX_WORKSPACE_QUOTA` | | Shell sessions | 16 per Runtime | `SANDBOX_MAX_SHELL_SESSIONS` | diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index f783149..87bacab 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -12,9 +12,10 @@ export KUBECONFIG="$PWD/.sandbox/kubeconfig" ## `make doctor` cannot reach Docker or reports insufficient capacity **Symptom.** The first quickstart phase stops before creating a VM because the -Docker daemon is unreachable, available memory is below 6.5 GiB, or free space under -`$LIMA_HOME` is below 25 GiB for a new profile. Reusing an existing `sandbox-local` -profile lowers the capacity gate to 2 GiB memory and 5 GiB disk. +Docker daemon is unreachable, available memory is below 10.5 GiB, or free space under +`$LIMA_HOME` is below 35 GiB for the default new profile. Reusing all requested +nodes lowers the capacity gate to 2 GiB memory and 5 GiB disk; expanding the pool +adds a capacity gate for every missing worker. **Fix.** On macOS, start Docker Desktop (`open -a Docker`) and wait until it is ready. On Linux, start the Docker service and verify `docker context show`. Free @@ -64,7 +65,7 @@ kubectl -n sandbox-workloads describe pod | sed -n '/Events/,$p' ``` `describe` shows `FailedCreatePodSandBox ... runsc` when the handler is missing on the -node and `0/1 nodes are available: node(s) didn't match Pod's node affinity/selector` +node and `0/N nodes are available: node(s) didn't match Pod's node affinity/selector` when the label is missing. **Fix.** In the local profile, rerun `make up-local`; it re-applies the label and the @@ -73,6 +74,41 @@ clusters, install gVisor on the Runtime nodes, apply `k8s/platform/runtime-class and label the nodes to match `SANDBOX_RUNTIME_NODE_SELECTOR`. A RuntimeClass object alone is not proof that `runsc` works; `make status-local` and the E2E suite verify it. +## `make scale-workers` refuses to scale down + +**Symptom.** The command prints `active Runtime Pods are still scheduled there` +and leaves the worker pool unchanged. + +**Cause.** At least one worker selected for removal still owns a Runtime Pod. +Stopping that node would interrupt active execution, so the resize preflights all +targets, cordons and rechecks them as a set, then rolls back command-added cordons +if the resize is refused. + +**Fix.** List the affected Pods, release or checkpoint their Runtime state, and +retry the resize: + +```bash +kubectl -n sandbox-workloads get pods -l app.kubernetes.io/name=sandbox-runtime -o wide +make scale-workers WORKERS=N +``` + +Worker VM disks are retained on a successful scale-down and reused on scale-up. +The stopped worker's Kubernetes Node registration and orphaned DaemonSet Pods are +removed so Cilium and CSI rollouts do not wait on a machine that is intentionally off. + +## Worker cannot reach the kubeadm API + +**Symptom.** `kubeadm join` waits or reports that the control-plane address on +port 6443 is unreachable. Both VMs may show the same default Lima guest address. + +**Cause.** The VMs were created on Lima's isolated default usernet rather than +the profile-owned `user-v2` network. Separate usernet instances cannot route to +each other even when their displayed addresses match. + +**Fix.** Current installers create `-net` and verify it before +reusing a VM. Preserve any legacy Workspace data, then destroy and recreate the +local profile. Do not bypass the network compatibility check. + ## `ErrImageNeverPull` or `ImagePullBackOff` **Symptom.** Control Plane, console, volume agent, or Runtime Pods show @@ -89,8 +125,9 @@ template, which must be reachable from the node. **Check.** ```bash -docker image ls | grep -E 'sandbox-(runtime|file-service|control plane|console)' +docker image ls | grep -E 'sandbox-(runtime|file-service|control-plane|console)' limactl shell sandbox-local -- sudo ctr --namespace k8s.io images ls | grep sandbox- +limactl shell sandbox-local-w1 -- sudo ctr --namespace k8s.io images ls | grep sandbox- kubectl -n sandbox-system describe pod | sed -n '/Events/,$p' ``` @@ -99,6 +136,34 @@ so they pick up the new image under the same tag. For a custom Runtime template, make sure the registry is allowed by `SANDBOX_IMAGE_REGISTRIES` and reachable from the node (see the network section below). +## Existing Workspace volume does not use `sandbox-rwx` + +**Symptom.** Upgrading an older local profile stops before applying manifests because +the existing Workspace PVC uses `local-path` instead of `sandbox-rwx`. + +**Cause.** The former single-node profile bound Workspace data to one VM-local +directory. Kubernetes cannot convert that PVC to CephFS in place, and silently +recreating it would lose data. + +**Fix.** Export or checkpoint any data you need first. Then run +`make destroy-local && make quickstart` to create the scalable profile from scratch. +The installer deliberately refuses this destructive migration. + +## Existing `sandbox-rwx` StorageClass has incompatible parameters + +**Symptom.** `make up-local` reports that `sandbox-rwx` needs the FUSE mounter, +or Kubernetes rejects a StorageClass update because its parameters are immutable. + +**Cause.** An earlier local profile created `sandbox-rwx` before the CephFS +userspace-mounter requirement was recorded. Kubernetes cannot mutate the +parameters of an existing StorageClass. + +**Fix.** If no PV or PVC references the class, rerun `make up-local`; the installer +removes and recreates only the unused StorageClass. If any volume references it, +the installer deliberately stops. Export or checkpoint Workspace data, then follow +the migration guidance printed by the command. Never delete the PVC or PV merely +to make the installer continue. + ## `SANDBOX_TOKEN is required; local tool fallback is disabled` **Symptom.** `sandbox`, the SDK, or `sandbox-mcp` exits immediately with this message. diff --git a/docs/index.html b/docs/index.html index 9cdc385..0d60ada 100644 --- a/docs/index.html +++ b/docs/index.html @@ -6,12 +6,12 @@ @@ -45,7 +45,7 @@

Agent execution
without the leap of faith.

Run untrusted shell and file operations on your own Kubernetes cluster. - Every runtime is a gVisor Pod. Every workspace survives it. Nothing falls back to your host. + Every runtime is a gVisor Pod. Every workspace survives it. Scale workers from zero to N without rebuilding the control plane.

Run it locally @@ -55,6 +55,7 @@

Agent execution
without the leap of faith.

MIT licensed Self-hosted kubeadm + Elastic workers No vendor runtime
@@ -71,6 +72,7 @@

Agent execution
without the leap of faith.

$ make quickstart

[1/5] checking host prerequisites

✓ kubeadm cluster ready

+

✓ Runtime worker pool ready

✓ gVisor kernel verified

✓ workspace survived restart

✓ host fallback blocked

@@ -86,7 +88,7 @@

Agent execution
without the leap of faith.

2.497s

cold start p50

35.72ms

warm execution p50

-
839

contract tests

+
844

contract tests

0

host fallbacks

@@ -120,13 +122,13 @@

A narrow waist between your agent and everything else.

default-deny NetworkPolicy
- UNTRUSTED · SANDBOX-WORKLOADS + UNTRUSTED · GVISOR WORKER POOL · 0…N
gVisor RuntimeClass - Ephemeral Runtime Pod + Ephemeral Runtime Pods × N
non-rootread-only rootno SA token
-
Durable WorkspacePVC · independent lifecycle
+
Durable WorkspaceCephFS RWX · survives worker churn
@@ -162,6 +164,18 @@

No privileged shortcuts

No Docker socket, no service-account token, non-root processes, and a read-only root filesystem.

kernel = gVisor +
+ 05 +

Elastic Runtime workers

+

Add workers without rebuilding the control plane. Scale-down preflights every target and refuses to strand an active Runtime.

+ make scale-workers WORKERS=N +
+
+ 06 +

One cluster. No hidden coupling.

+

A standalone kubeadm topology with ordinary labels, taints, RuntimeClass, and RWX storage. No infrastructure repository is imported.

+ control plane + worker pool +
@@ -193,7 +207,7 @@

Fast enough to stay in the agent loop.

29.21 ms p50 -

Transparent scope: local, single-node, warm images. Not presented as a cloud or node-cold result.

+

Transparent scope: local, multi-node kubeadm, warm images. Not presented as a cloud or node-cold result.

@@ -204,8 +218,9 @@

Fast enough to stay in the agent loop.

From clone to proof

One command. A real kubeadm cluster.

- The quickstart creates an isolated Lima VM, installs Kubernetes and gVisor, - deploys the platform, and exercises the security and persistence contract. + The quickstart creates a control-plane VM plus a scalable Runtime worker pool, installs kubeadm and gVisor, + deploys the platform, and exercises the security and persistence contract. Resize later with + make scale-workers WORKERS=N.

diff --git a/overlays/local-dev/kustomization.yaml b/overlays/local-dev/kustomization.yaml index 1c8707e..8b1be89 100644 --- a/overlays/local-dev/kustomization.yaml +++ b/overlays/local-dev/kustomization.yaml @@ -8,4 +8,3 @@ resources: patches: - path: control-plane-sqlite.yaml - - path: workspace-storage-local-path.yaml diff --git a/overlays/local-dev/workspace-storage-local-path.yaml b/overlays/local-dev/workspace-storage-local-path.yaml index c3ca282..b944d17 100644 --- a/overlays/local-dev/workspace-storage-local-path.yaml +++ b/overlays/local-dev/workspace-storage-local-path.yaml @@ -1,3 +1,6 @@ +# Legacy patch retained for operators composing the local-dev component with an +# RWO-only profile. The scalable overlays/local profile does not reference it; +# that profile uses the sandbox-rwx CephFS StorageClass. apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/overlays/local/kustomization.yaml b/overlays/local/kustomization.yaml index 9183fce..e456d1e 100644 --- a/overlays/local/kustomization.yaml +++ b/overlays/local/kustomization.yaml @@ -2,11 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization # Local integration profile. Runtime Pods use the installed gVisor RuntimeClass. -# SQLite and local-path workspace/object PVCs keep the profile self-contained; +# SQLite and CephFS workspace/object storage keep the profile self-contained; # they are not a production durability topology. Control-plane state or objects # hosted outside the cluster are an explicit opt-in: see overlays/external-deps. resources: - - ../rwo-single-node + - ../../k8s components: - ../local-dev diff --git a/rook/ceph-csi-drivers-values.yaml b/rook/ceph-csi-drivers-values.yaml new file mode 100644 index 0000000..e54de47 --- /dev/null +++ b/rook/ceph-csi-drivers-values.yaml @@ -0,0 +1,49 @@ +# Rook-compatible values for the independently managed Ceph-CSI drivers. +# Keep this aligned with the pinned Rook release. Rook 1.20 installs the CSI +# operator, but deliberately no longer creates Driver resources itself. +operatorConfig: + namespace: rook-ceph + driverSpecDefaults: + imageSet: + name: rook-csi-operator-image-set-configmap + nodePlugin: + priorityClassName: system-node-critical + # Runtime workers are intentionally tainted. CSI is node infrastructure, + # so it must still register there before CephFS-backed Runtimes can mount. + tolerations: + - key: sandbox.hullwork.com/node-role + operator: Equal + value: runtime + effect: NoSchedule + controllerPlugin: + priorityClassName: system-cluster-critical + +drivers: + rbd: + enabled: true + name: rook-ceph.rbd.csi.ceph.com + nodePlugin: + tolerations: + - key: sandbox.hullwork.com/node-role + operator: Equal + value: runtime + effect: NoSchedule + cephfs: + enabled: true + name: rook-ceph.cephfs.csi.ceph.com + # Ubuntu Noble's 6.8 kernel cannot parse Ceph 20.2.4 AES256K keys. Let the + # security-fixed v3.17.1 client select its userspace fallback instead of + # forcing the incompatible kernel path or weakening CephX to legacy AES. + cephFsClientType: autodetect + nodePlugin: + tolerations: + - key: sandbox.hullwork.com/node-role + operator: Equal + value: runtime + effect: NoSchedule + nfs: + enabled: false + name: rook-ceph.nfs.csi.ceph.com + nvmeof: + enabled: false + name: rook-ceph.nvmeof.csi.ceph.com diff --git a/rook/cluster-local.yaml b/rook/cluster-local.yaml index e49a067..43fc677 100644 --- a/rook/cluster-local.yaml +++ b/rook/cluster-local.yaml @@ -22,12 +22,15 @@ spec: crashCollector: disable: true storage: - useAllNodes: true + # The local Runtime worker is intentionally compute-only. The installer + # substitutes the trusted control-plane node name before applying this + # manifest, so Rook never probes or consumes a Runtime-worker device. + useAllNodes: false useAllDevices: false - # Rook v1.20 deliberately excludes loop devices from deviceFilter matches. - # The local preparation DaemonSet deterministically owns this exact device. - devices: - - name: /dev/loop0 + nodes: + - name: __SANDBOX_CONTROL_PLANE_NODE__ + devices: + - name: /dev/loop0 disruptionManagement: managePodBudgets: false priorityClassNames: @@ -71,3 +74,47 @@ spec: displayName: Sandbox Runtime quotas: maxBuckets: 3 +--- +apiVersion: ceph.rook.io/v1 +kind: CephFilesystem +metadata: + name: sandbox-filesystem + namespace: rook-ceph +spec: + metadataPool: + replicated: + size: 1 + requireSafeReplicaSize: false + dataPools: + - name: replicated + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + metadataServer: + activeCount: 1 + activeStandby: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: sandbox-rwx +provisioner: rook-ceph.cephfs.csi.ceph.com +parameters: + clusterID: rook-ceph + fsName: sandbox-filesystem + pool: sandbox-filesystem-replicated + # Lima's Ubuntu Noble kernel is 6.8 and cannot consume Ceph 20.2.4 AES256K + # keys. Keep secure keys and use the matching v3.17.1 userspace client. + mounter: fuse + csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-publish-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph +reclaimPolicy: Retain +allowVolumeExpansion: true +volumeBindingMode: Immediate diff --git a/scripts/dev-doctor.sh b/scripts/dev-doctor.sh index db8f273..754a640 100755 --- a/scripts/dev-doctor.sh +++ b/scripts/dev-doctor.sh @@ -35,9 +35,11 @@ PY if ! docker info >/dev/null 2>&1; then printf 'Docker is installed but its daemon is not reachable.\n' >&2 - if [ "$(uname -s)" = Darwin; then + if [ "$(uname -s)" = Darwin ]; then printf 'Start Docker Desktop (for example: open -a Docker), wait for it to report Ready, then rerun make doctor.\n' >&2 else + # Backticks are documentation in this single-quoted message. + # shellcheck disable=SC2016 printf 'Start the Docker service and verify `docker context show` points at the intended daemon, then rerun make doctor.\n' >&2 fi exit 1 @@ -56,27 +58,69 @@ if [ "$(uname -s)" = Linux ]; then fi fi -# A new default VM reserves 6 GiB of memory and a 60 GiB disk (sparse) -# (scripts/local-cluster.sh). The reference clean install consumes under 15 GiB -# of physical host storage; 6.5/25 keeps a real safety margin without rejecting a -# host that can run the profile merely because the sparse virtual size is 60 GiB. -# Reusing an existing VM only needs enough headroom to build images and roll -# Pods; applying the new-VM threshold after a successful install made retries -# fail precisely because the VM was already consuming those resources. +# A new default profile reserves 6 GiB for the control-plane VM and 4 GiB for +# the Runtime worker. Their 60/30 GiB disks are sparse. Reusing both nodes only +# needs enough headroom to build images and roll Pods; adding the worker to a +# legacy single-node profile uses the smaller expansion gate. # Set SANDBOX_DOCTOR_SKIP_RESOURCES=1 to bypass either mode. if [ "${SANDBOX_DOCTOR_SKIP_RESOURCES:-0}" = 1 ]; then printf 'resources check skipped (SANDBOX_DOCTOR_SKIP_RESOURCES=1)\n' else LIMA_DISK_DIR="${LIMA_HOME:-$HOME/.lima}" [ -d "$LIMA_DISK_DIR" ] || LIMA_DISK_DIR="$HOME" - DEFAULT_MIN_MEMORY_GIB=6.5 - DEFAULT_MIN_DISK_GIB=25 - RESOURCE_MODE=new-profile - if limactl list --format '{{.Name}}' 2>/dev/null \ - | grep -Fx "${SANDBOX_LOCAL_VM:-sandbox-local}" >/dev/null; then + CONTROL_PLANE_VM="${SANDBOX_LOCAL_VM:-sandbox-local}" + RUNTIME_VM_PREFIX="${SANDBOX_LOCAL_WORKER_PREFIX:-${CONTROL_PLANE_VM}-w}" + RUNTIME_WORKER_COUNT="${SANDBOX_LOCAL_WORKER_COUNT:-1}" + CONTROL_PLANE_MEMORY_GIB="${SANDBOX_LOCAL_MEMORY_GIB:-6}" + CONTROL_PLANE_DISK_GIB="${SANDBOX_LOCAL_DISK_GIB:-60}" + RUNTIME_MEMORY_GIB="${SANDBOX_LOCAL_WORKER_MEMORY_GIB:-4}" + RUNTIME_DISK_GIB="${SANDBOX_LOCAL_WORKER_DISK_GIB:-30}" + [[ "$RUNTIME_WORKER_COUNT" =~ ^[1-9][0-9]*$ ]] || { + printf 'SANDBOX_LOCAL_WORKER_COUNT must be a positive integer, got %s\n' \ + "$RUNTIME_WORKER_COUNT" >&2 + exit 1 + } + for RESOURCE_VALUE in "$CONTROL_PLANE_MEMORY_GIB" "$CONTROL_PLANE_DISK_GIB" \ + "$RUNTIME_MEMORY_GIB" "$RUNTIME_DISK_GIB"; do + [[ "$RESOURCE_VALUE" =~ ^[0-9]+([.][0-9]+)?$ ]] || { + printf 'VM memory and disk values must be positive numbers, got %s\n' \ + "$RESOURCE_VALUE" >&2 + exit 1 + } + done + RESOURCE_RESERVES="$(python3 - "$CONTROL_PLANE_MEMORY_GIB" \ + "$CONTROL_PLANE_DISK_GIB" "$RUNTIME_MEMORY_GIB" "$RUNTIME_DISK_GIB" <<'PY' +import sys + +control_memory, control_disk, worker_memory, worker_disk = map(float, sys.argv[1:]) +if min(control_memory, control_disk, worker_memory, worker_disk) <= 0: + raise SystemExit("VM memory and disk values must be greater than zero") +print(control_memory, min(control_disk, 25), worker_memory, min(worker_disk, 10)) +PY +)" + read -r CONTROL_MEMORY_RESERVE CONTROL_DISK_RESERVE \ + WORKER_MEMORY_RESERVE WORKER_DISK_RESERVE <<<"$RESOURCE_RESERVES" + VM_NAMES="$(limactl list --format '{{.Name}}' 2>/dev/null || true)" + CONTROL_PLANE_EXISTS=0 + grep -Fx "$CONTROL_PLANE_VM" <<<"$VM_NAMES" >/dev/null && CONTROL_PLANE_EXISTS=1 + EXISTING_WORKERS=0 + for WORKER_INDEX in $(seq 1 "$RUNTIME_WORKER_COUNT"); do + grep -Fx "${RUNTIME_VM_PREFIX}${WORKER_INDEX}" <<<"$VM_NAMES" >/dev/null \ + && EXISTING_WORKERS=$((EXISTING_WORKERS + 1)) + done + MISSING_WORKERS=$((RUNTIME_WORKER_COUNT - EXISTING_WORKERS)) + if ((CONTROL_PLANE_EXISTS && MISSING_WORKERS == 0)); then DEFAULT_MIN_MEMORY_GIB=2 DEFAULT_MIN_DISK_GIB=5 RESOURCE_MODE=reuse-profile + elif ((CONTROL_PLANE_EXISTS)); then + DEFAULT_MIN_MEMORY_GIB="$(python3 -c "print(2 + $WORKER_MEMORY_RESERVE * $MISSING_WORKERS + 0.5)")" + DEFAULT_MIN_DISK_GIB="$(python3 -c "print(5 + $WORKER_DISK_RESERVE * $MISSING_WORKERS)")" + RESOURCE_MODE=expand-worker-pool + else + DEFAULT_MIN_MEMORY_GIB="$(python3 -c "print($CONTROL_MEMORY_RESERVE + $WORKER_MEMORY_RESERVE * $MISSING_WORKERS + 0.5)")" + DEFAULT_MIN_DISK_GIB="$(python3 -c "print($CONTROL_DISK_RESERVE + $WORKER_DISK_RESERVE * $MISSING_WORKERS)")" + RESOURCE_MODE=new-profile fi SANDBOX_DOCTOR_DEFAULT_MIN_MEMORY_GIB="$DEFAULT_MIN_MEMORY_GIB" \ SANDBOX_DOCTOR_DEFAULT_MIN_DISK_GIB="$DEFAULT_MIN_DISK_GIB" \ diff --git a/scripts/install-cilium-kubeadm.sh b/scripts/install-cilium-kubeadm.sh index 9f1dd70..84a654b 100755 --- a/scripts/install-cilium-kubeadm.sh +++ b/scripts/install-cilium-kubeadm.sh @@ -6,7 +6,6 @@ CLUSTER="${1:?usage: install-cilium-kubeadm.sh }" CILIUM_VERSION="${CILIUM_VERSION:-1.19.6}" CILIUM_CHART_SHA256="${CILIUM_CHART_SHA256:-21c43cf53841f9ab0375047d95aa4c64051ea52bbd2c679416e6408f5f1c9179}" -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" : "${KUBECONFIG:?set KUBECONFIG to the target kubeadm cluster configuration}" # The default-value branch creates an ordinary shell variable that Helm cannot # see in its child process; export KUBECONFIG explicitly. @@ -41,8 +40,9 @@ if [[ "${ACTUAL_CHART_SHA256}" != "${CILIUM_CHART_SHA256}" ]]; then exit 1 fi -# Two operator replicas on a single-node cluster cannot satisfy anti-affinity. -#helm --wait; --set-string prevents helm from parsing false as a boolean instead of a string. +# One operator replica keeps the local profile lightweight as workers scale up +# and down; the Cilium agent itself remains a DaemonSet on every active node. +# helm --wait; --set-string prevents Helm from parsing false as a boolean. helm --kube-context "${CLUSTER}" upgrade --install cilium "${CHART_FILE}" \ --namespace kube-system \ --set routingMode=tunnel --set tunnelProtocol=vxlan \ diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 4f1d70b..f877ddc 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -3,17 +3,25 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -VM_NAME="${SANDBOX_LOCAL_VM:-sandbox-local}" +CONTROL_PLANE_VM="${SANDBOX_LOCAL_VM:-sandbox-local}" +RUNTIME_VM_PREFIX="${SANDBOX_LOCAL_WORKER_PREFIX:-${CONTROL_PLANE_VM}-w}" +RUNTIME_WORKER_COUNT="${SANDBOX_LOCAL_WORKER_COUNT:-1}" +CLUSTER_NETWORK="${SANDBOX_LOCAL_NETWORK:-${CONTROL_PLANE_VM}-net}" +CLUSTER_NETWORK_GATEWAY="${SANDBOX_LOCAL_NETWORK_GATEWAY:-}" STATE_DIR="${SANDBOX_STATE_DIR:-$REPO_ROOT/.sandbox}" SANDBOX_KUBECONFIG="${SANDBOX_KUBECONFIG:-$STATE_DIR/kubeconfig}" SANDBOX_KUBE_CONTEXT="${SANDBOX_KUBE_CONTEXT:-sandbox-local}" SANDBOX_POD_CIDR="${SANDBOX_POD_CIDR:-10.244.0.0/16}" API_PORT="${SANDBOX_LOCAL_API_PORT:-18448}" SANDBOX_CONTROL_PLANE_PORT="${SANDBOX_LOCAL_CONTROL_PLANE_PORT:-18080}" -VM_CPUS="${SANDBOX_LOCAL_CPUS:-4}" -VM_MEMORY_GIB="${SANDBOX_LOCAL_MEMORY_GIB:-6}" -VM_DISK_GIB="${SANDBOX_LOCAL_DISK_GIB:-60}" +CONTROL_PLANE_CPUS="${SANDBOX_LOCAL_CPUS:-4}" +CONTROL_PLANE_MEMORY_GIB="${SANDBOX_LOCAL_MEMORY_GIB:-6}" +CONTROL_PLANE_DISK_GIB="${SANDBOX_LOCAL_DISK_GIB:-60}" +RUNTIME_CPUS="${SANDBOX_LOCAL_WORKER_CPUS:-4}" +RUNTIME_MEMORY_GIB="${SANDBOX_LOCAL_WORKER_MEMORY_GIB:-4}" +RUNTIME_DISK_GIB="${SANDBOX_LOCAL_WORKER_DISK_GIB:-30}" IMAGE_BUILD_JOBS="${SANDBOX_IMAGE_BUILD_JOBS:-4}" +CAPACITY_RESTART_REQUIRED=0 # Fixed local tags: imagePullPolicy Never in the manifests matches these names. PROJECT_IMAGES=( sandbox-runtime:0.5.0 @@ -24,13 +32,73 @@ PROJECT_IMAGES=( usage() { printf '%s\n' \ - "usage: scripts/local-cluster.sh " \ + "usage: scripts/local-cluster.sh " \ "" \ - "up create/reuse the Lima VM and deploy the local gVisor profile" \ + "up create/reuse the 1+N Lima/kubeadm cluster and deploy gVisor" \ "status show nodes, RuntimeClass, and sandbox workloads" \ - "down stop the VM without deleting its disk" \ - "destroy delete the VM disk, the state directory, and local images" \ - "kubeconfig print the isolated kubeconfig path" + "down stop the control plane and every worker without deleting disks" \ + "destroy delete all profile VM disks, the state directory, and local images" \ + "kubeconfig print the isolated kubeconfig path" \ + "scale-workers COUNT safely scale the Runtime worker pool (COUNT may be 0)" +} + +runtime_vm_name() { + printf '%s%s\n' "$RUNTIME_VM_PREFIX" "$1" +} + +runtime_vms() { + limactl list --format '{{.Name}}' 2>/dev/null \ + | awk -v prefix="$RUNTIME_VM_PREFIX" \ + 'index($0, prefix) == 1 && substr($0, length(prefix) + 1) ~ /^[1-9][0-9]*$/ { print }' +} + +network_exists() { + limactl network list --json 2>/dev/null | python3 -c 'import json,sys +wanted = sys.argv[1] +raise SystemExit(0 if any(json.loads(line).get("name") == wanted for line in sys.stdin if line.strip()) else 1)' \ + "$CLUSTER_NETWORK" +} + +ensure_cluster_network() { + if ! network_exists; then + local gateway + gateway="$(limactl network list --json 2>/dev/null | python3 -c 'import ipaddress,json,sys +requested = sys.argv[1] +if requested: + interface = ipaddress.ip_interface(requested) + if interface.version != 4 or interface.network.prefixlen != 24: + raise SystemExit("SANDBOX_LOCAL_NETWORK_GATEWAY must be an IPv4 /24 gateway") + print(str(interface)) + raise SystemExit(0) +used = {item.get("gateway") for line in sys.stdin if line.strip() for item in (json.loads(line),)} +for octet in range(108, 224): + candidate = f"192.168.{octet}.1" + if candidate not in used: + print(f"{candidate}/24") + raise SystemExit(0) +raise SystemExit("no free Lima user-v2 subnet in 192.168.108.0/24..192.168.223.0/24")' \ + "$CLUSTER_NETWORK_GATEWAY")" + limactl network create "$CLUSTER_NETWORK" --mode=user-v2 --gateway="$gateway" + fi +} + +configured_vm_network() { + local vm="$1" + limactl list "$vm" --json | python3 -c 'import json,sys +networks = json.load(sys.stdin).get("config", {}).get("networks") or [] +print(next((item.get("lima", "") for item in networks if item.get("lima")), ""))' +} + +verify_vm_network() { + local vm="$1" + local configured + configured="$(configured_vm_network "$vm")" + if [ "$configured" != "$CLUSTER_NETWORK" ]; then + printf '%s\n' \ + "Lima VM $vm uses network ${configured:-isolated-usernet}, not the shared cluster network $CLUSTER_NETWORK." \ + "The installer will not recreate an existing VM. Preserve any data, run 'make destroy-local', then rerun 'make quickstart'." >&2 + return 1 + fi } require_tools() { @@ -44,28 +112,35 @@ require_tools() { docker info >/dev/null local port for port in "$API_PORT" "$SANDBOX_CONTROL_PLANE_PORT"; do - [[ "$port" =~ ^[0-9]+$ ]] && (( port >= 1024 && port <= 65535 )) || { + if ! [[ "$port" =~ ^[0-9]+$ ]] || ((port < 1024 || port > 65535)); then printf 'invalid Lima host port: %s\n' "$port" >&2 return 1 - } + fi done [ "$API_PORT" != "$SANDBOX_CONTROL_PLANE_PORT" ] || { printf 'SANDBOX_LOCAL_API_PORT and SANDBOX_LOCAL_CONTROL_PLANE_PORT must differ\n' >&2 return 1 } + [[ "$RUNTIME_WORKER_COUNT" =~ ^[1-9][0-9]*$ ]] || { + printf 'SANDBOX_LOCAL_WORKER_COUNT must be a positive integer for up, got %s\n' \ + "$RUNTIME_WORKER_COUNT" >&2 + return 1 + } } vm_exists() { - limactl list --format '{{.Name}}' 2>/dev/null | grep -Fx "$VM_NAME" >/dev/null + local vm="$1" + limactl list --format '{{.Name}}' 2>/dev/null | grep -Fx "$vm" >/dev/null } vm_running() { - [ "$(limactl list "$VM_NAME" --format '{{.Status}}' 2>/dev/null)" = Running ] + local vm="$1" + [ "$(limactl list "$vm" --format '{{.Status}}' 2>/dev/null)" = Running ] } configured_host_port() { local guest_port="$1" - limactl list "$VM_NAME" --json \ + limactl list "$CONTROL_PLANE_VM" --json \ | python3 -c 'import json,sys guest = int(sys.argv[1]) config = json.load(sys.stdin).get("config", {}) @@ -79,22 +154,57 @@ verify_existing_ports() { configured_control_plane="$(configured_host_port 30080)" if [ "$configured_api" != "$API_PORT" ] || [ "$configured_control_plane" != "$SANDBOX_CONTROL_PLANE_PORT" ]; then printf '%s\n' \ - "Lima VM $VM_NAME already uses API/Control Plane ports ${configured_api:-unknown}/${configured_control_plane:-unknown};" \ + "Lima VM $CONTROL_PLANE_VM already uses API/Control Plane ports ${configured_api:-unknown}/${configured_control_plane:-unknown};" \ "requested $API_PORT/$SANDBOX_CONTROL_PLANE_PORT. Reuse its configured ports or choose a new SANDBOX_LOCAL_VM." >&2 return 1 fi } guest_ip() { - limactl shell "$VM_NAME" -- sh -c \ + local vm="$1" + limactl shell "$vm" -- sh -c \ "ip -4 route get 1.1.1.1 | sed -n 's/.* src \\([^ ]*\\).*/\\1/p'" } +guest_hostname() { + local vm="$1" + limactl list "$vm" --json | python3 -c \ + 'import json,sys; print(json.load(sys.stdin)["hostname"])' +} + +remove_worker_node_registration() { + local runtime_node="$1" + # The VM is already stopped (or known to be stopped), so force-removing its + # remaining DaemonSet Pods cannot create duplicate processes. Node deletion + # alone leaves those Pods counted as unavailable for several minutes, which + # makes a subsequent Helm --wait look hung. + kubectl --context "$SANDBOX_KUBE_CONTEXT" delete node "$runtime_node" \ + --ignore-not-found --wait=true >/dev/null + kubectl --context "$SANDBOX_KUBE_CONTEXT" delete pod --all-namespaces \ + --field-selector="spec.nodeName=$runtime_node" \ + --grace-period=0 --force --wait=false >/dev/null 2>&1 || true +} + +remove_stale_worker_nodes() { + local runtime_vm runtime_node + while read -r runtime_vm; do + [ -n "$runtime_vm" ] || continue + vm_running "$runtime_vm" && continue + runtime_node="$(guest_hostname "$runtime_vm")" + if kubectl --context "$SANDBOX_KUBE_CONTEXT" get node "$runtime_node" \ + >/dev/null 2>&1; then + printf 'removing stopped worker %s from the live Kubernetes node set\n' \ + "$runtime_node" + fi + remove_worker_node_registration "$runtime_node" + done < <(runtime_vms) +} + write_kubeconfig() { local api_ip="$1" mkdir -p "$STATE_DIR" umask 077 - limactl shell "$VM_NAME" -- sudo cat /etc/kubernetes/admin.conf >"$SANDBOX_KUBECONFIG" + limactl shell "$CONTROL_PLANE_VM" -- sudo cat /etc/kubernetes/admin.conf >"$SANDBOX_KUBECONFIG" KUBECONFIG="$SANDBOX_KUBECONFIG" kubectl config rename-context \ kubernetes-admin@kubernetes "$SANDBOX_KUBE_CONTEXT" >/dev/null 2>&1 || true KUBECONFIG="$SANDBOX_KUBECONFIG" kubectl config set-cluster kubernetes \ @@ -103,45 +213,126 @@ write_kubeconfig() { chmod 600 "$SANDBOX_KUBECONFIG" } +wait_for_kubernetes_api() { + local attempt + for ((attempt = 1; attempt <= 90; attempt++)); do + if kubectl --context "$SANDBOX_KUBE_CONTEXT" get --raw=/readyz \ + >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + printf 'Kubernetes API did not become ready within 180 seconds\n' >&2 + return 1 +} + +ensure_kubeadm_prerequisites() { + local vm="$1" + if limactl shell "$vm" -- sh -c \ + 'command -v kubeadm >/dev/null && command -v kubelet >/dev/null && command -v ctr >/dev/null'; then + return 0 + fi + printf 'Kubernetes prerequisites are incomplete on %s; retrying the idempotent provisioner\n' "$vm" + # Lima currently reports a VM as ready even when a system provision script + # exits non-zero. Re-running the generated script turns that silent partial + # boot into either a recovered node or an explicit installer failure. + limactl shell "$vm" -- sudo sh /mnt/lima-cidata/provision.system/00000000 + limactl shell "$vm" -- sh -c \ + 'command -v kubeadm >/dev/null && command -v kubelet >/dev/null && command -v ctr >/dev/null' +} + +ensure_runtime_worker() { + local worker_index="$1" + local runtime_vm runtime_node join_command + runtime_vm="$(runtime_vm_name "$worker_index")" + if ! vm_exists "$runtime_vm"; then + limactl start --name "$runtime_vm" --tty=false \ + --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY_GIB" \ + --disk "$RUNTIME_DISK_GIB" \ + --network "lima:$CLUSTER_NETWORK" \ + --set '.portForwards = []' \ + "$SCRIPT_DIR/local-cluster.yaml" + else + verify_vm_network "$runtime_vm" + if ! vm_running "$runtime_vm"; then + limactl start --tty=false "$runtime_vm" + fi + fi + ensure_kubeadm_prerequisites "$runtime_vm" + runtime_node="$(guest_hostname "$runtime_vm")" + if ! kubectl --context "$SANDBOX_KUBE_CONTEXT" get node "$runtime_node" >/dev/null 2>&1; then + if limactl shell "$runtime_vm" -- sudo test -s /etc/kubernetes/kubelet.conf; then + limactl shell "$runtime_vm" -- sudo kubeadm reset --force >/dev/null + fi + join_command="$(limactl shell "$CONTROL_PLANE_VM" -- sudo kubeadm token create \ + --ttl 30m --print-join-command)" + limactl shell "$runtime_vm" -- sudo sh -c "$join_command" + # The command is intentionally single-quoted for evaluation in the guest. + # shellcheck disable=SC2016 + limactl shell "$CONTROL_PLANE_VM" -- sudo bash -c \ + 'kubeadm token list 2>/dev/null | awk "NR > 1 {print \$1}" | xargs -r kubeadm token delete >/dev/null' + fi + # Keep the node unschedulable until gVisor is installed and the Runtime image + # has been imported. A Runtime Pod selects the role label below, so labeling + # an uncordoned node first would create a race with image/runtime readiness. + kubectl --context "$SANDBOX_KUBE_CONTEXT" cordon "$runtime_node" >/dev/null + kubectl --context "$SANDBOX_KUBE_CONTEXT" label node "$runtime_node" \ + sandbox.hullwork.com/node-role=runtime --overwrite >/dev/null + kubectl --context "$SANDBOX_KUBE_CONTEXT" taint node "$runtime_node" \ + sandbox.hullwork.com/node-role=runtime:NoSchedule --overwrite >/dev/null + bash "$SCRIPT_DIR/install-gvisor-kubeadm.sh" "$runtime_vm" + kubectl --context "$SANDBOX_KUBE_CONTEXT" wait node "$runtime_node" \ + --for=condition=Ready --timeout=5m +} + ensure_cluster() { - local api_ip - if ! vm_exists; then - limactl start --name "$VM_NAME" --tty=false \ - --cpus "$VM_CPUS" --memory "$VM_MEMORY_GIB" --disk "$VM_DISK_GIB" \ + local api_ip control_plane_node worker_index + if ! vm_exists "$CONTROL_PLANE_VM"; then + ensure_cluster_network + limactl start --name "$CONTROL_PLANE_VM" --tty=false \ + --cpus "$CONTROL_PLANE_CPUS" --memory "$CONTROL_PLANE_MEMORY_GIB" \ + --disk "$CONTROL_PLANE_DISK_GIB" \ + --network "lima:$CLUSTER_NETWORK" \ --set ".portForwards[0].hostPort = $API_PORT" \ --set ".portForwards[1].hostPort = $SANDBOX_CONTROL_PLANE_PORT" \ "$SCRIPT_DIR/local-cluster.yaml" else + verify_vm_network "$CONTROL_PLANE_VM" verify_existing_ports fi - if ! vm_running; then - limactl start --tty=false "$VM_NAME" + if ! vm_running "$CONTROL_PLANE_VM"; then + limactl start --tty=false "$CONTROL_PLANE_VM" fi - api_ip="$(guest_ip)" + ensure_kubeadm_prerequisites "$CONTROL_PLANE_VM" + api_ip="$(guest_ip "$CONTROL_PLANE_VM")" [ -n "$api_ip" ] || { echo 'cannot determine Lima guest IP' >&2; return 1; } - if ! limactl shell "$VM_NAME" -- sudo test -s /etc/kubernetes/admin.conf; then - limactl shell "$VM_NAME" -- sudo kubeadm init \ + if ! limactl shell "$CONTROL_PLANE_VM" -- sudo test -s /etc/kubernetes/admin.conf; then + limactl shell "$CONTROL_PLANE_VM" -- sudo kubeadm init \ --apiserver-advertise-address="$api_ip" \ --apiserver-cert-extra-sans=127.0.0.1 \ --pod-network-cidr="$SANDBOX_POD_CIDR" \ --skip-token-print 2>&1 \ | sed -E 's/[a-z0-9]{6}\.[a-z0-9]{16}/[redacted-bootstrap-token]/g' - # The bootstrap-token phase also creates kubelet API RBAC used by logs and - # exec. Keep that phase, but remove its short-lived join credential because - # this profile is intentionally single-node. - limactl shell "$VM_NAME" -- sudo bash -c \ - 'kubeadm token list 2>/dev/null | awk "NR > 1 {print \$1}" | xargs -r kubeadm token delete >/dev/null' fi write_kubeconfig "$api_ip" export KUBECONFIG="$SANDBOX_KUBECONFIG" + wait_for_kubernetes_api + # A retained, stopped Lima disk must not leave a NotReady Kubernetes Node + # behind: Helm waits on node-wide DaemonSets such as Cilium and CSI. Delete + # only that registration; ensure_runtime_worker reuses the disk and lets the + # kubelet register again when the worker is requested later. + remove_stale_worker_nodes bash "$SCRIPT_DIR/install-cilium-kubeadm.sh" "$SANDBOX_KUBE_CONTEXT" - kubectl --context "$SANDBOX_KUBE_CONTEXT" taint nodes --all \ + control_plane_node="$(guest_hostname "$CONTROL_PLANE_VM")" + kubectl --context "$SANDBOX_KUBE_CONTEXT" taint node "$control_plane_node" \ node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true - kubectl --context "$SANDBOX_KUBE_CONTEXT" label nodes --all \ - sandbox.hullwork.com/node-role=runtime --overwrite >/dev/null - bash "$SCRIPT_DIR/install-gvisor-kubeadm.sh" "$VM_NAME" - kubectl --context "$SANDBOX_KUBE_CONTEXT" wait node --all \ + kubectl --context "$SANDBOX_KUBE_CONTEXT" label node "$control_plane_node" \ + sandbox.hullwork.com/node-role=system --overwrite >/dev/null + kubectl --context "$SANDBOX_KUBE_CONTEXT" wait node "$control_plane_node" \ --for=condition=Ready --timeout=5m + for ((worker_index = 1; worker_index <= RUNTIME_WORKER_COUNT; worker_index++)); do + ensure_runtime_worker "$worker_index" + done } build_images() { @@ -153,20 +344,83 @@ build_images() { make -C "$REPO_ROOT" --no-print-directory -j "$IMAGE_BUILD_JOBS" images } +load_image() { + local image="$1" + local vm="$2" + printf ' loading %s into %s\n' "$image" "$vm" + docker image inspect "$image" >/dev/null 2>&1 || docker pull "$image" + docker save "$image" | limactl shell "$vm" -- sudo ctr \ + --namespace k8s.io images import - >/dev/null +} + load_images() { - local image - for image in \ - "${PROJECT_IMAGES[@]}" \ - registry.k8s.io/metrics-server/metrics-server@sha256:d9862115e7c7881280d3d75ca26bda8ffc0fc213315979575bf23ce9826205c0; do - printf ' loading %s into %s\n' "$image" "$VM_NAME" - docker image inspect "$image" >/dev/null 2>&1 || docker pull "$image" - docker save "$image" | limactl shell "$VM_NAME" -- sudo ctr \ - --namespace k8s.io images import - >/dev/null + # System images never need to enter an untrusted Runtime worker. The Control + # Plane image also serves the volume-agent role, which stays on the trusted + # system node. Keeping the placement list explicit makes image transfer part + # of the same topology contract as the Kubernetes scheduling rules. + load_image sandbox-control-plane:0.7.0 "$CONTROL_PLANE_VM" + load_image sandbox-console:0.1.0 "$CONTROL_PLANE_VM" + load_image sandbox-file-service:0.3.0 "$CONTROL_PLANE_VM" + load_image \ + registry.k8s.io/metrics-server/metrics-server@sha256:d9862115e7c7881280d3d75ca26bda8ffc0fc213315979575bf23ce9826205c0 \ + "$CONTROL_PLANE_VM" + local worker_index + for ((worker_index = 1; worker_index <= RUNTIME_WORKER_COUNT; worker_index++)); do + local runtime_vm runtime_node + runtime_vm="$(runtime_vm_name "$worker_index")" + load_image sandbox-runtime:0.5.0 "$runtime_vm" + runtime_node="$(guest_hostname "$runtime_vm")" + kubectl --context "$SANDBOX_KUBE_CONTEXT" uncordon "$runtime_node" >/dev/null done } +wait_for_ceph_csi() { + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace rook-ceph rollout status \ + deployment/rook-ceph.cephfs.csi.ceph.com-ctrlplugin --timeout=10m + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace rook-ceph rollout status \ + daemonset/rook-ceph.cephfs.csi.ceph.com-nodeplugin --timeout=10m + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace rook-ceph rollout status \ + deployment/rook-ceph.rbd.csi.ceph.com-ctrlplugin --timeout=10m + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace rook-ceph rollout status \ + daemonset/rook-ceph.rbd.csi.ceph.com-nodeplugin --timeout=10m +} + +reconcile_workspace_storage_class() { + local current_mounter referenced_pvs referenced_pvcs + current_mounter="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" get storageclass sandbox-rwx \ + -o jsonpath='{.parameters.mounter}' 2>/dev/null || true)" + if [ -z "$current_mounter" ]; then + if ! kubectl --context "$SANDBOX_KUBE_CONTEXT" get storageclass sandbox-rwx \ + >/dev/null 2>&1; then + return 0 + fi + elif [ "$current_mounter" = fuse ]; then + return 0 + fi + + # StorageClass parameters are immutable. Recreate an obsolete class only + # when it is provably unused; existing volumes require an explicit migration + # so an installer rerun can never delete or strand user data. + referenced_pvs="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" get pv \ + -o jsonpath='{range .items[?(@.spec.storageClassName=="sandbox-rwx")]}{.metadata.name}{"\n"}{end}')" + referenced_pvcs="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" get pvc --all-namespaces \ + -o jsonpath='{range .items[?(@.spec.storageClassName=="sandbox-rwx")]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}')" + if [ -n "$referenced_pvs" ] || [ -n "$referenced_pvcs" ]; then + printf '%s\n' \ + "existing sandbox-rwx volumes use an obsolete immutable StorageClass configuration (mounter=${current_mounter:-unset})." \ + "The installer will not replace it while PVs or PVCs exist. Back up and migrate those volumes before rerunning quickstart." \ + "PVs: ${referenced_pvs:-none}" \ + "PVCs: ${referenced_pvcs:-none}" >&2 + return 1 + fi + printf 'recreating unused sandbox-rwx StorageClass with mounter=fuse\n' + kubectl --context "$SANDBOX_KUBE_CONTEXT" delete storageclass sandbox-rwx +} + deploy_ceph_rgw() { export KUBECONFIG="$SANDBOX_KUBECONFIG" + local control_plane_node + control_plane_node="$(guest_hostname "$CONTROL_PLANE_VM")" kubectl --context "$SANDBOX_KUBE_CONTEXT" create namespace rook-ceph \ --dry-run=client -o yaml \ | kubectl --context "$SANDBOX_KUBE_CONTEXT" label --local -f - \ @@ -179,6 +433,9 @@ deploy_ceph_rgw() { # ties the tarball that arrives to the one that was reviewed. local rook_version="${ROOK_CEPH_CHART_VERSION:-v1.20.6}" local rook_sha256="${ROOK_CEPH_CHART_SHA256:-83a16ee19dd8d621df4159504b33585d80da1bf7ed83c734a9e8d4828c724353}" + # v3.17.0 embeds Ceph 20.2.1 and cannot decode the AES256K keys generated by + # security-fixed Ceph 20.2.4. v3.17.1 is the upstream compatibility release. + local ceph_csi_image="${CEPH_CSI_IMAGE:-v3.17.1@sha256:0b62db8afc9b174619186c87084a8952157daa719415c598603be22b63cc1293}" local rook_dir rook_chart rook_actual rook_dir="$(mktemp -d "${TMPDIR:-/tmp}/sandbox-rook-chart.XXXXXX")" helm pull rook-ceph --repo https://charts.rook.io/release \ @@ -194,18 +451,50 @@ deploy_ceph_rgw() { --namespace rook-ceph \ --kubeconfig "$SANDBOX_KUBECONFIG" \ --set csi.installCsiOperator=true \ + --set-string "csi.cephcsi.tag=$ceph_csi_image" \ --set enableDiscoveryDaemon=false \ --set allowLoopDevices=true \ --wait --timeout 10m rm -rf "$rook_dir" + # Rook 1.20 delegates actual Driver resources to a second chart. Installing + # only rook-ceph leaves StorageClasses present but no CSI provisioner, so a + # clean quickstart would wait forever on its first PVC. + local csi_version="${CEPH_CSI_DRIVERS_CHART_VERSION:-1.0.4}" + local csi_sha256="${CEPH_CSI_DRIVERS_CHART_SHA256:-76a1787baa7d62232eb073ab8260a455a016c02b59aae584a47be6791f05994b}" + local csi_dir csi_chart csi_actual + csi_dir="$(mktemp -d "${TMPDIR:-/tmp}/sandbox-ceph-csi-chart.XXXXXX")" + helm pull ceph-csi-drivers --repo https://ceph.github.io/ceph-csi-operator \ + --version "$csi_version" --destination "$csi_dir" + csi_chart="$csi_dir/ceph-csi-drivers-$csi_version.tgz" + csi_actual="$(shasum -a 256 "$csi_chart" | awk '{print $1}')" + if [ "$csi_actual" != "$csi_sha256" ]; then + printf 'Ceph-CSI drivers chart checksum mismatch: expected %s, got %s\n' \ + "$csi_sha256" "$csi_actual" >&2 + rm -rf "$csi_dir" + return 1 + fi + helm upgrade --install ceph-csi-drivers "$csi_chart" \ + --namespace rook-ceph \ + --kubeconfig "$SANDBOX_KUBECONFIG" \ + --values "$REPO_ROOT/rook/ceph-csi-drivers-values.yaml" \ + --wait --timeout 10m + rm -rf "$csi_dir" + # The drivers chart creates CRs; their operator-owned Deployments and + # DaemonSets are asynchronous and are not covered by Helm's --wait. + wait_for_ceph_csi kubectl --context "$SANDBOX_KUBE_CONTEXT" apply -f "$REPO_ROOT/rook/loop-device.yaml" kubectl --context "$SANDBOX_KUBE_CONTEXT" -n rook-ceph rollout status \ daemonset/rook-local-loop-device --timeout=5m - kubectl --context "$SANDBOX_KUBE_CONTEXT" apply -f "$REPO_ROOT/rook/cluster-local.yaml" + reconcile_workspace_storage_class + sed "s/__SANDBOX_CONTROL_PLANE_NODE__/$control_plane_node/g" \ + "$REPO_ROOT/rook/cluster-local.yaml" \ + | kubectl --context "$SANDBOX_KUBE_CONTEXT" apply -f - kubectl --context "$SANDBOX_KUBE_CONTEXT" -n rook-ceph wait \ --for=condition=Ready cephcluster/rook-ceph --timeout=20m kubectl --context "$SANDBOX_KUBE_CONTEXT" -n rook-ceph wait \ "--for=jsonpath={.status.phase}=Ready" cephobjectstore/object-store --timeout=15m + kubectl --context "$SANDBOX_KUBE_CONTEXT" -n rook-ceph wait \ + "--for=jsonpath={.status.phase}=Ready" cephfilesystem/sandbox-filesystem --timeout=15m local rook_secret=rook-ceph-object-user-object-store-sandbox-runtime local access_key secret_key for _ in $(seq 1 120); do @@ -260,6 +549,197 @@ sandbox-workloads sandbox-workspace-init EOF } +verify_workspace_storage_compatibility() { + local storage_class + storage_class="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" \ + --namespace sandbox-workloads get pvc sandbox-workspaces \ + -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)" + if [ -n "$storage_class" ] && [ "$storage_class" != sandbox-rwx ]; then + printf '%s\n' \ + "existing local Workspace volume uses StorageClass $storage_class, not the scalable sandbox-rwx CephFS class." \ + "The installer will not silently delete or move Workspace data. Back it up, run 'make destroy-local', then rerun 'make quickstart'." >&2 + return 1 + fi +} + +assert_pods_on_node() { + local namespace="$1" + local selector="$2" + local expected_node="$3" + local actual_nodes + actual_nodes="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace "$namespace" \ + get pods -l "$selector" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}')" + [ -n "$actual_nodes" ] || { + printf 'no Pods found for %s/%s\n' "$namespace" "$selector" >&2 + return 1 + } + if grep -Fvx "$expected_node" <<<"$actual_nodes" >/dev/null; then + printf 'Pods for %s/%s are not confined to %s: %s\n' \ + "$namespace" "$selector" "$expected_node" "${actual_nodes//$'\n'/, }" >&2 + return 1 + fi +} + +verify_deployed_topology() { + local control_plane_node runtime_node worker_index + control_plane_node="$(guest_hostname "$CONTROL_PLANE_VM")" + assert_pods_on_node sandbox-system \ + app.kubernetes.io/name=sandbox-control-plane "$control_plane_node" + assert_pods_on_node sandbox-system \ + app.kubernetes.io/name=sandbox-console "$control_plane_node" + assert_pods_on_node sandbox-workloads \ + app.kubernetes.io/name=sandbox-volume "$control_plane_node" + verify_workspace_storage_compatibility + for ((worker_index = 1; worker_index <= RUNTIME_WORKER_COUNT; worker_index++)); do + runtime_node="$(guest_hostname "$(runtime_vm_name "$worker_index")")" + [ "$control_plane_node" != "$runtime_node" ] + [ "$(kubectl --context "$SANDBOX_KUBE_CONTEXT" get node "$runtime_node" \ + -o jsonpath='{.metadata.labels.sandbox\.hullwork\.com/node-role}')" = runtime ] + [ "$(kubectl --context "$SANDBOX_KUBE_CONTEXT" get node "$runtime_node" \ + -o jsonpath='{.spec.taints[?(@.key=="sandbox.hullwork.com/node-role")].value}')" = runtime ] + done + printf 'Topology verified: system=%s, active Runtime workers=%s\n' \ + "$control_plane_node" "$RUNTIME_WORKER_COUNT" +} + +scale_local_capacity() { + local quota_workers max_runtimes pods services requests_cpu requests_memory + local limits_cpu limits_memory previous_max_runtimes + # Runtime admission follows the active worker count exactly, including a + # deliberate scale-to-zero. Keep a one-worker-sized namespace floor for the + # trusted volume agent and initialization Jobs that also live in + # sandbox-workloads. + quota_workers="$RUNTIME_WORKER_COUNT" + ((quota_workers > 0)) || quota_workers=1 + max_runtimes=$((4 * RUNTIME_WORKER_COUNT)) + pods=$((10 + max_runtimes)) + services=$((8 + max_runtimes)) + requests_cpu=$((2 * quota_workers)) + requests_memory=$((1300 * quota_workers)) + limits_cpu=$((4 * quota_workers)) + limits_memory=$((3328 * quota_workers)) + previous_max_runtimes="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" \ + --namespace sandbox-system get configmap sandbox-tuning \ + -o jsonpath='{.data.SANDBOX_MAX_RUNTIMES}' 2>/dev/null || true)" + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace sandbox-system \ + patch configmap sandbox-tuning --type merge \ + -p "{\"data\":{\"SANDBOX_MAX_RUNTIMES\":\"$max_runtimes\",\"SANDBOX_MAX_INFLIGHT_CREATES\":\"$((2 * RUNTIME_WORKER_COUNT))\"}}" >/dev/null + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace sandbox-workloads \ + patch resourcequota sandbox-workload-quota --type merge \ + -p "{\"spec\":{\"hard\":{\"pods\":\"$pods\",\"services\":\"$services\",\"requests.cpu\":\"$requests_cpu\",\"requests.memory\":\"${requests_memory}Mi\",\"limits.cpu\":\"$limits_cpu\",\"limits.memory\":\"${limits_memory}Mi\"}}}" >/dev/null + CAPACITY_RESTART_REQUIRED=0 + [ "$previous_max_runtimes" = "$max_runtimes" ] || CAPACITY_RESTART_REQUIRED=1 + printf 'Local Runtime capacity: %s workers, %s concurrent Runtimes\n' \ + "$RUNTIME_WORKER_COUNT" "$max_runtimes" +} + +scale_workers() { + local desired="$1" + local runtime_vm runtime_node worker_index active_runtime_pods was_unschedulable + local -a workers_to_stop=() + local -a nodes_to_uncordon_on_failure=() + [[ "$desired" =~ ^[0-9]+$ ]] || { + printf 'worker count must be a non-negative integer, got %s\n' "$desired" >&2 + return 2 + } + if ((desired > 0)); then + SANDBOX_LOCAL_WORKER_COUNT="$desired" bash "$SCRIPT_DIR/dev-doctor.sh" + fi + RUNTIME_WORKER_COUNT="$desired" + ensure_cluster + if kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace rook-ceph \ + get driver.csi.ceph.io/rook-ceph.cephfs.csi.ceph.com >/dev/null 2>&1; then + wait_for_ceph_csi + fi + if ((desired > 0)); then + make -C "$REPO_ROOT" --no-print-directory image-runtime + for ((worker_index = 1; worker_index <= desired; worker_index++)); do + runtime_vm="$(runtime_vm_name "$worker_index")" + load_image sandbox-runtime:0.5.0 "$runtime_vm" + runtime_node="$(guest_hostname "$runtime_vm")" + kubectl --context "$SANDBOX_KUBE_CONTEXT" uncordon "$runtime_node" >/dev/null + done + fi + # Preflight every target before changing any node. If one target still owns + # a Runtime, the requested resize is rejected atomically instead of stopping + # some higher-numbered workers and failing halfway through. + while read -r runtime_vm; do + [ -n "$runtime_vm" ] || continue + worker_index="${runtime_vm#"$RUNTIME_VM_PREFIX"}" + ((worker_index > desired)) || continue + if ! vm_running "$runtime_vm"; then + continue + fi + runtime_node="$(guest_hostname "$runtime_vm")" + active_runtime_pods="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" \ + --namespace sandbox-workloads get pods \ + -l app.kubernetes.io/name=sandbox-runtime \ + --field-selector="spec.nodeName=$runtime_node" --no-headers 2>/dev/null || true)" + if [ -n "$active_runtime_pods" ]; then + printf 'refusing to stop %s: active Runtime Pods are still scheduled there\n%s\n' \ + "$runtime_vm" "$active_runtime_pods" >&2 + return 1 + fi + workers_to_stop+=("$runtime_vm") + done < <(runtime_vms | sort -Vr) + # Close the scheduler race after the read-only preflight. New admissions can + # still happen between that check and cordon, so cordon every target and then + # check them all again before the first drain/stop. + # Bash 3.2 on macOS treats an empty array expansion as unset under `set -u`. + # Keep every expansion behind a length check so scale-up-only operations do + # not fail after the new workers are already healthy. + if ((${#workers_to_stop[@]})); then + for runtime_vm in "${workers_to_stop[@]}"; do + runtime_node="$(guest_hostname "$runtime_vm")" + was_unschedulable="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" get node \ + "$runtime_node" -o jsonpath='{.spec.unschedulable}')" + if [ "$was_unschedulable" != true ]; then + kubectl --context "$SANDBOX_KUBE_CONTEXT" cordon "$runtime_node" >/dev/null + nodes_to_uncordon_on_failure+=("$runtime_node") + fi + done + for runtime_vm in "${workers_to_stop[@]}"; do + runtime_node="$(guest_hostname "$runtime_vm")" + active_runtime_pods="$(kubectl --context "$SANDBOX_KUBE_CONTEXT" \ + --namespace sandbox-workloads get pods \ + -l app.kubernetes.io/name=sandbox-runtime \ + --field-selector="spec.nodeName=$runtime_node" --no-headers 2>/dev/null || true)" + if [ -n "$active_runtime_pods" ]; then + local node_to_uncordon + if ((${#nodes_to_uncordon_on_failure[@]})); then + for node_to_uncordon in "${nodes_to_uncordon_on_failure[@]}"; do + kubectl --context "$SANDBOX_KUBE_CONTEXT" uncordon "$node_to_uncordon" \ + >/dev/null 2>&1 || true + done + fi + printf 'refusing to stop %s: a Runtime Pod reached the node during scale-down preflight\n%s\n' \ + "$runtime_vm" "$active_runtime_pods" >&2 + return 1 + fi + done + for runtime_vm in "${workers_to_stop[@]}"; do + runtime_node="$(guest_hostname "$runtime_vm")" + kubectl --context "$SANDBOX_KUBE_CONTEXT" drain "$runtime_node" \ + --ignore-daemonsets --delete-emptydir-data --force --timeout=3m >/dev/null + limactl stop "$runtime_vm" + remove_worker_node_registration "$runtime_node" + done + fi + if kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace sandbox-system \ + get configmap sandbox-tuning >/dev/null 2>&1; then + scale_local_capacity + if ((CAPACITY_RESTART_REQUIRED)); then + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace sandbox-system \ + rollout restart deployment/sandbox-control-plane >/dev/null + kubectl --context "$SANDBOX_KUBE_CONTEXT" --namespace sandbox-system \ + rollout status deployment/sandbox-control-plane --timeout=4m + fi + else + printf 'Control Plane is not deployed yet; run make up-local to apply capacity.\n' + fi + printf 'Runtime worker pool scaled to %s active node(s)\n' "$desired" +} + apply_local_overlay() { local output result output="$(mktemp "${TMPDIR:-/tmp}/sandbox-kustomize.XXXXXX")" @@ -286,6 +766,7 @@ deploy_local_profile() { local system_deployments_to_restart=() local restart_volume_deployment=0 local deployment image + verify_workspace_storage_compatibility for deployment in sandbox-control-plane sandbox-console; do if [ "$deployment" = sandbox-control-plane ]; then image=sandbox-control-plane:0.7.0 @@ -304,10 +785,6 @@ deploy_local_profile() { sandbox-control-plane:0.7.0; then restart_volume_deployment=1 fi - kubectl --context "$SANDBOX_KUBE_CONTEXT" apply -f \ - https://raw.githubusercontent.com/rancher/local-path-provisioner/c4fdcada94c2e632cd7d9231e73406d554eb40e2/deploy/local-path-storage.yaml - kubectl --context "$SANDBOX_KUBE_CONTEXT" -n local-path-storage rollout status \ - deployment/local-path-provisioner --timeout=3m kubectl --context "$SANDBOX_KUBE_CONTEXT" apply -f "$REPO_ROOT/k8s/namespaces.yaml" SANDBOX_KUBE_CONTEXT="$SANDBOX_KUBE_CONTEXT" bash "$SCRIPT_DIR/bootstrap-local-secrets.sh" deploy_ceph_rgw @@ -318,6 +795,11 @@ deploy_local_profile() { # path in apply_local_overlay. delete_failed_initialization_jobs apply_local_overlay + scale_local_capacity + if ((CAPACITY_RESTART_REQUIRED)) \ + && [[ ! " ${system_deployments_to_restart[*]} " =~ " deployment/sandbox-control-plane " ]]; then + system_deployments_to_restart+=("deployment/sandbox-control-plane") + fi # Project images use fixed local tags and imagePullPolicy Never. Importing a # new image under the same tag does not change the Deployment template, so an # existing Pod would otherwise keep running the previous image indefinitely. @@ -347,6 +829,7 @@ deploy_local_profile() { deployment/metrics-server --timeout=3m kubectl --context "$SANDBOX_KUBE_CONTEXT" wait --for=condition=Available \ apiservice/v1beta1.metrics.k8s.io --timeout=3m + verify_deployed_topology } destroy_local_profile() { @@ -358,8 +841,18 @@ destroy_local_profile() { return 1 ;; esac - printf 'deleting Lima VM %s\n' "$VM_NAME" - limactl delete -f "$VM_NAME" || true + local runtime_vm + while read -r runtime_vm; do + [ -n "$runtime_vm" ] || continue + printf 'deleting Lima VM %s\n' "$runtime_vm" + limactl delete -f "$runtime_vm" || true + done < <(runtime_vms) + printf 'deleting Lima VM %s\n' "$CONTROL_PLANE_VM" + limactl delete -f "$CONTROL_PLANE_VM" || true + if network_exists; then + printf 'deleting Lima network %s\n' "$CLUSTER_NETWORK" + limactl network delete --force "$CLUSTER_NETWORK" || true + fi printf 'deleting state directory %s\n' "$STATE_DIR" rm -rf "$STATE_DIR" || true local image @@ -403,23 +896,28 @@ run_up_phase() { case "${1:-}" in up) run_up_phase 'checking required tools and ports' require_tools - run_up_phase 'creating or resuming the Lima Kubernetes node' ensure_cluster + run_up_phase 'creating or resuming the control plane and Runtime worker pool' ensure_cluster run_up_phase "building 4 project images ($IMAGE_BUILD_JOBS parallel jobs)" build_images - run_up_phase 'loading project and Metrics Server images into containerd' load_images + run_up_phase 'loading role-scoped images into Kubernetes nodes' load_images run_up_phase 'deploying storage, Control Plane, Runtime services, and Console' deploy_local_profile printf 'Control Plane: http://127.0.0.1:%s\nKubeconfig: %s\n' \ "$SANDBOX_CONTROL_PLANE_PORT" "$SANDBOX_KUBECONFIG" ;; status) export KUBECONFIG="$SANDBOX_KUBECONFIG" - kubectl --context "$SANDBOX_KUBE_CONTEXT" get nodes + kubectl --context "$SANDBOX_KUBE_CONTEXT" get nodes \ + -L sandbox.hullwork.com/node-role kubectl --context "$SANDBOX_KUBE_CONTEXT" get runtimeclass gvisor kubectl --context "$SANDBOX_KUBE_CONTEXT" -n kube-system get deployment metrics-server kubectl --context "$SANDBOX_KUBE_CONTEXT" get apiservice v1beta1.metrics.k8s.io kubectl --context "$SANDBOX_KUBE_CONTEXT" get pods -A -l app.kubernetes.io/managed-by=sandbox-controller ;; down) - vm_exists && limactl stop "$VM_NAME" + while read -r runtime_vm; do + [ -n "$runtime_vm" ] || continue + vm_running "$runtime_vm" && limactl stop "$runtime_vm" + done < <(runtime_vms | sort -Vr) + vm_exists "$CONTROL_PLANE_VM" && limactl stop "$CONTROL_PLANE_VM" ;; destroy) destroy_local_profile @@ -427,5 +925,9 @@ case "${1:-}" in kubeconfig) printf '%s\n' "$SANDBOX_KUBECONFIG" ;; + scale-workers) + [ "$#" -eq 2 ] || { usage >&2; exit 2; } + scale_workers "$2" + ;; *) usage >&2; exit 2 ;; esac diff --git a/scripts/local-cluster.yaml b/scripts/local-cluster.yaml index ba7dabc..a01a540 100644 --- a/scripts/local-cluster.yaml +++ b/scripts/local-cluster.yaml @@ -39,8 +39,11 @@ provision: EOF sysctl --system export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq containerd apt-transport-https ca-certificates curl gpg + # A newly-created Lima user-v2 NIC can become usable a few seconds before + # its DNS proxy is ready. Retry transient fetch failures so a successful + # VM boot cannot leave kubeadm half-installed. + apt-get -o Acquire::Retries=5 update -qq + apt-get -o Acquire::Retries=5 install -y -qq containerd apt-transport-https ca-certificates curl gpg mkdir -p /etc/containerd /etc/apt/keyrings containerd config default >/etc/containerd/config.toml sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml @@ -49,10 +52,10 @@ provision: # its first boot rather than caching containerd's built-in cgroupfs default. systemctl enable containerd systemctl restart containerd - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.36/deb/Release.key \ + curl --retry 5 --retry-all-errors --retry-delay 2 -fsSL https://pkgs.k8s.io/core:/stable:/v1.36/deb/Release.key \ | gpg --dearmor --yes -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.36/deb/ /' \ >/etc/apt/sources.list.d/kubernetes.list - apt-get update -qq - apt-get install -y -qq kubelet=1.36.* kubeadm=1.36.* kubectl=1.36.* + apt-get -o Acquire::Retries=5 update -qq + apt-get -o Acquire::Retries=5 install -y -qq kubelet=1.36.* kubeadm=1.36.* kubectl=1.36.* apt-mark hold kubelet kubeadm kubectl diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index b9a4e4b..ef1ad13 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -11,10 +11,28 @@ CURRENT_PHASE=initialization PROFILE_MODE=created PHASE_RESULTS=() -if command -v limactl >/dev/null 2>&1 \ - && limactl list --format '{{.Name}}' 2>/dev/null \ - | grep -Fx "${SANDBOX_LOCAL_VM:-sandbox-local}" >/dev/null; then - PROFILE_MODE=reused +# The Lima host port is configurable, so the value proof must follow the same +# endpoint unless the caller explicitly supplied a different API URL. +export SANDBOX_CONTROL_PLANE_URL="${SANDBOX_CONTROL_PLANE_URL:-http://127.0.0.1:${SANDBOX_LOCAL_CONTROL_PLANE_PORT:-18080}}" + +if command -v limactl >/dev/null 2>&1; then + CONTROL_PLANE_VM="${SANDBOX_LOCAL_VM:-sandbox-local}" + RUNTIME_VM_PREFIX="${SANDBOX_LOCAL_WORKER_PREFIX:-${CONTROL_PLANE_VM}-w}" + RUNTIME_WORKER_COUNT="${SANDBOX_LOCAL_WORKER_COUNT:-1}" + VM_NAMES="$(limactl list --format '{{.Name}}' 2>/dev/null || true)" + EXISTING_WORKERS=0 + if [[ "$RUNTIME_WORKER_COUNT" =~ ^[1-9][0-9]*$ ]]; then + for WORKER_INDEX in $(seq 1 "$RUNTIME_WORKER_COUNT"); do + grep -Fx "${RUNTIME_VM_PREFIX}${WORKER_INDEX}" <<<"$VM_NAMES" >/dev/null \ + && EXISTING_WORKERS=$((EXISTING_WORKERS + 1)) + done + fi + if grep -Fx "$CONTROL_PLANE_VM" <<<"$VM_NAMES" >/dev/null \ + && [ "$EXISTING_WORKERS" = "$RUNTIME_WORKER_COUNT" ]; then + PROFILE_MODE=reused + elif grep -Fx "$CONTROL_PLANE_VM" <<<"$VM_NAMES" >/dev/null; then + PROFILE_MODE=expanded + fi fi mkdir -p "$STATE_DIR" diff --git a/scripts/test.sh b/scripts/test.sh index 2f16b0a..9df23c6 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -197,6 +197,12 @@ runtime_class="$( get pod "runtime-${SANDBOX_ID}" \ --output=jsonpath='{.spec.runtimeClassName}' )" +runtime_node="$( + kubectl --context "${SANDBOX_KUBE_CONTEXT}" \ + --namespace sandbox-workloads \ + get pod "runtime-${SANDBOX_ID}" \ + --output=jsonpath='{.spec.nodeName}' +)" runtime_subpath="$( kubectl --context "${SANDBOX_KUBE_CONTEXT}" \ --namespace sandbox-workloads \ @@ -220,6 +226,17 @@ runtime_containers="$( #When empty, jsonpath cannot obtain the field and returns an empty string, which is exactly equal to the expected value of empty - at the same time #It proves that there is indeed no runtimeClassName key in the Pod spec. test "$runtime_class" = "${SANDBOX_RUNTIME_CLASS}" +if [ "${SANDBOX_EXPECT_DEDICATED_RUNTIME_NODE:-0}" = 1 ]; then + test "$(kubectl --context "${SANDBOX_KUBE_CONTEXT}" get node "$runtime_node" \ + --output=jsonpath='{.metadata.labels.sandbox\.hullwork\.com/node-role}')" = runtime + control_plane_node="$( + kubectl --context "${SANDBOX_KUBE_CONTEXT}" --namespace sandbox-system \ + get pod -l app.kubernetes.io/name=sandbox-control-plane \ + --output=jsonpath='{.items[0].spec.nodeName}' + )" + test -n "$control_plane_node" + test "$runtime_node" != "$control_plane_node" +fi case "$workspace_storage_mode" in shared) test "$runtime_subpath" = "$WORKSPACE_ID" ;; per-workspace) test -z "$runtime_subpath" ;; diff --git a/tests/test_dev_scripts.py b/tests/test_dev_scripts.py index ab9d9d1..32611b7 100644 --- a/tests/test_dev_scripts.py +++ b/tests/test_dev_scripts.py @@ -35,9 +35,9 @@ def fake_tool_directory(directory: str, *, omit: tuple[str, ...] = ()) -> str: return directory -def make_dry_run(target: str) -> str: +def make_dry_run(target: str, *variables: str) -> str: return subprocess.run( - ["make", "--no-print-directory", "-n", target], + ["make", "--no-print-directory", "-n", target, *variables], cwd=ROOT, check=True, capture_output=True, @@ -205,6 +205,10 @@ def test_quickstart_uses_the_standalone_kubeadm_profile(self) -> None: combined = "\n".join((makefile, quickstart, cluster)) self.assertIn("scripts/local-cluster.sh up", makefile) self.assertIn("kubeadm init", cluster) + self.assertIn( + "${SANDBOX_LOCAL_CONTROL_PLANE_PORT:-18080}", + quickstart, + ) for forbidden in ( "kind create cluster", "kind delete cluster", @@ -213,6 +217,83 @@ def test_quickstart_uses_the_standalone_kubeadm_profile(self) -> None: ): self.assertNotIn(forbidden, combined) + def test_local_kubeadm_profile_has_a_scalable_runtime_worker_pool(self) -> None: + script = (ROOT / "scripts/local-cluster.sh").read_text(encoding="utf-8") + lima = (ROOT / "scripts/local-cluster.yaml").read_text(encoding="utf-8") + makefile = (ROOT / "Makefile").read_text(encoding="utf-8") + self.assertIn('RUNTIME_WORKER_COUNT="${SANDBOX_LOCAL_WORKER_COUNT:-1}"', script) + self.assertIn('CLUSTER_NETWORK="${SANDBOX_LOCAL_NETWORK:-${CONTROL_PLANE_VM}-net}"', script) + self.assertIn('limactl network create "$CLUSTER_NETWORK"', script) + self.assertIn("--mode=user-v2 --gateway=", script) + self.assertIn("range(108, 224)", script) + self.assertIn("SANDBOX_LOCAL_NETWORK_GATEWAY", script) + self.assertIn('--network "lima:$CLUSTER_NETWORK"', script) + self.assertIn("verify_vm_network", script) + self.assertIn('limactl network delete --force "$CLUSTER_NETWORK"', script) + self.assertIn("kubeadm token create", script) + self.assertIn("ensure_kubeadm_prerequisites", script) + self.assertIn("/mnt/lima-cidata/provision.system/00000000", script) + self.assertIn("Acquire::Retries=5", lima) + self.assertIn("--retry-all-errors", lima) + self.assertIn("--print-join-command", script) + self.assertIn('sudo sh -c "$join_command"', script) + self.assertIn("scale-workers)", script) + self.assertIn("active Runtime Pods are still scheduled there", script) + self.assertIn("kubectl --context \"$SANDBOX_KUBE_CONTEXT\" drain", script) + self.assertIn("scale_local_capacity", script) + self.assertIn('SANDBOX_LOCAL_WORKER_COUNT="$desired"', script) + self.assertIn("wait_for_ceph_csi", script) + self.assertIn("daemonset/rook-ceph.cephfs.csi.ceph.com-nodeplugin", script) + self.assertIn("wait_for_kubernetes_api", script) + self.assertIn("remove_stale_worker_nodes", script) + self.assertIn("remove_worker_node_registration", script) + self.assertIn('delete node "$runtime_node"', script) + self.assertIn('--field-selector="spec.nodeName=$runtime_node"', script) + self.assertLess( + script.index('write_kubeconfig "$api_ip"'), + script.index("wait_for_kubernetes_api", script.index('write_kubeconfig "$api_ip"')), + ) + self.assertIn("SANDBOX_MAX_RUNTIMES", script) + self.assertIn("patch resourcequota sandbox-workload-quota", script) + self.assertIn("max_runtimes=$((4 * RUNTIME_WORKER_COUNT))", script) + self.assertNotIn('for worker_index in $(seq 1 "$RUNTIME_WORKER_COUNT")', script) + self.assertIn('local -a workers_to_stop=()', script) + self.assertIn('if ((${#workers_to_stop[@]})); then', script) + ensure_worker = script[script.index("ensure_runtime_worker()") : script.index("ensure_cluster()")] + self.assertLess( + ensure_worker.index('cordon "$runtime_node"'), + ensure_worker.index('label node "$runtime_node"'), + ) + self.assertLess( + script.index('workers_to_stop+=("$runtime_vm")'), + script.index('for runtime_vm in "${workers_to_stop[@]}"'), + ) + self.assertIn('nodes_to_uncordon_on_failure+=("$runtime_node")', script) + self.assertIn("a Runtime Pod reached the node during scale-down preflight", script) + doctor = (ROOT / "scripts/dev-doctor.sh").read_text(encoding="utf-8") + self.assertIn('CONTROL_PLANE_MEMORY_GIB="${SANDBOX_LOCAL_MEMORY_GIB:-6}"', doctor) + self.assertIn('RUNTIME_MEMORY_GIB="${SANDBOX_LOCAL_WORKER_MEMORY_GIB:-4}"', doctor) + self.assertIn("WORKER_MEMORY_RESERVE * $MISSING_WORKERS", doctor) + self.assertIn("CONTROL_DISK_RESERVE + $WORKER_DISK_RESERVE", doctor) + self.assertIn("make scale-workers", (ROOT / "README.md").read_text()) + self.assertIn("scale-workers:", makefile) + self.assertIn( + 'scripts/local-cluster.sh scale-workers "2"', + make_dry_run("scale-workers", "WORKERS=2"), + ) + + def test_local_profile_loads_only_runtime_image_on_workers(self) -> None: + script = (ROOT / "scripts/local-cluster.sh").read_text(encoding="utf-8") + worker_loop = script[script.index("load_images()") : script.index("deploy_ceph_rgw()")] + self.assertIn('runtime_vm="$(runtime_vm_name "$worker_index")"', worker_loop) + self.assertIn('load_image sandbox-runtime:0.5.0 "$runtime_vm"', worker_loop) + self.assertLess( + worker_loop.index('load_image sandbox-runtime:0.5.0 "$runtime_vm"'), + worker_loop.index('uncordon "$runtime_node"'), + ) + self.assertNotIn('sandbox-console:0.1.0 "$(runtime_vm_name', worker_loop) + self.assertNotIn('sandbox-control-plane:0.7.0 "$(runtime_vm_name', worker_loop) + def test_existing_secret_migrates_the_previous_control_token_without_rotation(self) -> None: with tempfile.TemporaryDirectory() as directory: fake_kubectl = pathlib.Path(directory) / "kubectl" diff --git a/tests/test_documentation.py b/tests/test_documentation.py index 7782205..73c6e94 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -119,18 +119,26 @@ def test_contributor_clone_path_matches_this_repository(self) -> None: def test_documented_vm_disk_matches_installer_default(self) -> None: script = (ROOT / "scripts/local-cluster.sh").read_text(encoding="utf-8") - match = re.search(r'VM_DISK_GIB="\$\{SANDBOX_LOCAL_DISK_GIB:-(\d+)\}"', script) - self.assertIsNotNone(match) - size = match.group(1) + control_match = re.search( + r'CONTROL_PLANE_DISK_GIB="\$\{SANDBOX_LOCAL_DISK_GIB:-(\d+)\}"', + script, + ) + worker_match = re.search( + r'RUNTIME_DISK_GIB="\$\{SANDBOX_LOCAL_WORKER_DISK_GIB:-(\d+)\}"', + script, + ) + self.assertIsNotNone(control_match) + self.assertIsNotNone(worker_match) + control_size = control_match.group(1) + worker_size = worker_match.group(1) readme = (ROOT / "README.md").read_text(encoding="utf-8") doctor = (ROOT / "scripts/dev-doctor.sh").read_text(encoding="utf-8") - self.assertIn(f"{size} GiB disk by default", readme) - self.assertIn(f"its {size} GiB disk", readme) - self.assertIn(f"a {size} GiB disk", doctor) - self.assertIn("DEFAULT_MIN_MEMORY_GIB=6.5", doctor) - self.assertIn("DEFAULT_MIN_DISK_GIB=25", doctor) - self.assertIn("**6.5 GiB free**", readme) - self.assertIn("**25 GiB free**", readme) + self.assertIn(f"a {control_size} GiB disk", readme) + self.assertIn(f"{worker_size} GiB disk", readme) + self.assertIn("DEFAULT_MIN_MEMORY_GIB=2", doctor) + self.assertIn("DEFAULT_MIN_DISK_GIB=5", doctor) + self.assertIn("**10.5 GiB free**", readme) + self.assertIn("**35 GiB free**", readme) def test_mysql_driver_documentation_matches_the_image(self) -> None: dockerfile = (ROOT / "control_plane/Dockerfile").read_text(encoding="utf-8") diff --git a/tests/test_reaper_behavior.py b/tests/test_reaper_behavior.py index 0224f43..4d970d9 100644 --- a/tests/test_reaper_behavior.py +++ b/tests/test_reaper_behavior.py @@ -785,6 +785,14 @@ def test_pool_with_room_admits_without_touching_pods(self) -> None: self.assertEqual(kube.deleted("pods"), []) self.assertEqual(kube.list_calls, 1) + def test_zero_capacity_rejects_without_creating_a_runtime(self) -> None: + helpers, kube = self.helpers([]) + with self.assertRaises(KubeError) as raised: + helpers["_admit_new_runtime"](0) + self.assertEqual(raised.exception.status, HTTPStatus.TOO_MANY_REQUESTS) + self.assertIn("(0/0)", str(raised.exception)) + self.assertEqual(kube.deleted("pods"), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_standalone_contract.py b/tests/test_standalone_contract.py index b2a6cf4..fef6694 100644 --- a/tests/test_standalone_contract.py +++ b/tests/test_standalone_contract.py @@ -199,14 +199,14 @@ def test_local_overlay_is_self_contained(self) -> None: ) self.assertIn("kind: Component", component) self.assertIn("control-plane-sqlite.yaml", component) - self.assertIn("workspace-storage-local-path.yaml", component) + self.assertNotIn("workspace-storage-local-path.yaml", component) self.assertIn("object-store.yaml", component) self.assertIn("metrics-server.yaml", component) # The local profile must consume that component rather than carry its # own copies or point at dependencies outside the cluster. local = (ROOT / "overlays/local/kustomization.yaml").read_text(encoding="utf-8") self.assertIn("components:\n - ../local-dev", local) - self.assertIn("resources:\n - ../rwo-single-node", local) + self.assertIn("resources:\n - ../../k8s", local) self.assertNotIn("patches:", local) self.assertNotIn("control-plane-external.yaml", local) self.assertEqual( diff --git a/tests/test_storage_modes.py b/tests/test_storage_modes.py index ce6af5d..3333b29 100644 --- a/tests/test_storage_modes.py +++ b/tests/test_storage_modes.py @@ -59,6 +59,35 @@ def test_rook_v1_20_installs_the_ceph_connection_crd_provider(self) -> None: text = (REPO_ROOT / "scripts/local-cluster.sh").read_text() self.assertIn("--set csi.installCsiOperator=true", text) self.assertNotIn("--set csi.installCsiOperator=false", text) + self.assertIn("helm pull ceph-csi-drivers", text) + self.assertIn("CEPH_CSI_DRIVERS_CHART_SHA256", text) + self.assertIn("v3.17.1@sha256:", text) + self.assertIn("rook/ceph-csi-drivers-values.yaml", text) + + values = yaml.safe_load( + (REPO_ROOT / "rook/ceph-csi-drivers-values.yaml").read_text() + ) + self.assertTrue(values["drivers"]["cephfs"]["enabled"]) + self.assertEqual( + values["drivers"]["cephfs"]["name"], + "rook-ceph.cephfs.csi.ceph.com", + ) + self.assertEqual( + values["drivers"]["cephfs"]["cephFsClientType"], "autodetect" + ) + self.assertEqual( + values["operatorConfig"]["driverSpecDefaults"]["nodePlugin"][ + "tolerations" + ][0]["key"], + "sandbox.hullwork.com/node-role", + ) + for driver in ("rbd", "cephfs"): + self.assertEqual( + values["drivers"][driver]["nodePlugin"]["tolerations"][0][ + "key" + ], + "sandbox.hullwork.com/node-role", + ) def test_local_rook_selects_its_loop_device_by_exact_path(self) -> None: cluster = next( @@ -68,14 +97,63 @@ def test_local_rook_selects_its_loop_device_by_exact_path(self) -> None: if document and document.get("kind") == "CephCluster" ) storage = cluster["spec"]["storage"] - self.assertEqual(storage["devices"], [{"name": "/dev/loop0"}]) + self.assertFalse(storage["useAllNodes"]) + self.assertEqual( + storage["nodes"], + [{"name": "__SANDBOX_CONTROL_PLANE_NODE__", "devices": [{"name": "/dev/loop0"}]}], + ) self.assertNotIn("deviceFilter", storage) + def test_local_workspace_storage_is_cephfs_rwx(self) -> None: + documents = [ + item for item in yaml.safe_load_all( + (REPO_ROOT / "rook/cluster-local.yaml").read_text() + ) + if item + ] + filesystem = next(item for item in documents if item["kind"] == "CephFilesystem") + storage_class = next(item for item in documents if item["kind"] == "StorageClass") + self.assertEqual(filesystem["metadata"]["name"], "sandbox-filesystem") + self.assertEqual(storage_class["metadata"]["name"], "sandbox-rwx") + self.assertEqual( + storage_class["provisioner"], "rook-ceph.cephfs.csi.ceph.com" + ) + self.assertEqual(storage_class["parameters"]["fsName"], "sandbox-filesystem") + self.assertEqual( + storage_class["parameters"]["pool"], + "sandbox-filesystem-replicated", + ) + self.assertEqual(storage_class["parameters"]["mounter"], "fuse") + self.assertEqual( + storage_class["parameters"][ + "csi.storage.k8s.io/controller-publish-secret-name" + ], + "rook-csi-cephfs-provisioner", + ) + local_overlay = (REPO_ROOT / "overlays/local/kustomization.yaml").read_text() + self.assertIn("resources:\n - ../../k8s", local_overlay) + self.assertNotIn("rwo-single-node", local_overlay) + def test_rook_object_store_waits_for_the_reported_phase(self) -> None: text = (REPO_ROOT / "scripts/local-cluster.sh").read_text() self.assertIn('"--for=jsonpath={.status.phase}=Ready" cephobjectstore/object-store', text) self.assertNotIn("--for=condition=Ready cephobjectstore", text) + def test_storage_class_upgrade_is_data_preserving(self) -> None: + text = (REPO_ROOT / "scripts/local-cluster.sh").read_text() + reconcile = text[ + text.index("reconcile_workspace_storage_class()") : + text.index("deploy_ceph_rgw()") + ] + self.assertIn("get pv", reconcile) + self.assertIn("get pvc --all-namespaces", reconcile) + self.assertIn("will not replace it while PVs or PVCs exist", reconcile) + self.assertLess( + reconcile.index('if [ -n "$referenced_pvs" ]'), + reconcile.index("delete storageclass sandbox-rwx"), + ) + self.assertIn("reconcile_workspace_storage_class", text) + def test_lima_restarts_fixed_tag_project_deployments_only_on_update(self) -> None: text = (REPO_ROOT / "scripts/local-cluster.sh").read_text() detection_at = text.index("system_deployments_to_restart=()") @@ -116,9 +194,6 @@ def test_lima_waits_only_for_objects_the_local_overlay_renders(self) -> None: r"-n (\S+) (?:rollout status|wait) \\\n\s+(?:--for=\S+ )?(deployment|job)/(\S+)", script, ) - # local-path-provisioner is applied from its upstream manifest, - # not from the overlay. - if namespace != "local-path-storage" ] self.assertGreaterEqual(len(awaited), 6) self.assertEqual([entry for entry in awaited if entry not in present], []) From 50613bb6604c489b9479b8df26c576e874c688e2 Mon Sep 17 00:00:00 2001 From: convee Date: Sat, 5 Sep 2026 12:15:26 +0800 Subject: [PATCH 2/2] test: avoid backtracking in release parser --- tests/test_release_assets.py | 37 +++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/test_release_assets.py b/tests/test_release_assets.py index 8a593a4..18f7ec7 100644 --- a/tests/test_release_assets.py +++ b/tests/test_release_assets.py @@ -5,6 +5,7 @@ import os import pathlib import re +import shlex import shutil import subprocess import tempfile @@ -60,12 +61,34 @@ def record_image_identity(component: str, directory: pathlib.Path) -> None: ) return # jq -n --arg a "$A" ... '{k:$a,...,immutableRef:($repository+"@"+$digest)}' - match = re.search( - r"jq -n((?:\s+--arg \w+ \"\$\w+\"\s*\\?)+)\s*'(\{.*?\})'\s*\\?\s*>\"(.*?)\"", - RECORD_STEP["run"], re.S, + lines = RECORD_STEP["run"].splitlines() + start = next( + (index for index, line in enumerate(lines) if line.lstrip().startswith("jq ")), + None, ) - assert match, "the jq invocation is not where this fallback looks" - arguments = dict(re.findall(r'--arg (\w+) "\$(\w+)"', match.group(1))) + assert start is not None, "the jq invocation is not where this fallback looks" + command_parts = [] + for line in lines[start:]: + part = line.strip() + continued = part.endswith("\\") + command_parts.append(part[:-1].strip() if continued else part) + if not continued: + break + command = " ".join(command_parts) + assert len(command) <= 4096, "the jq invocation is unexpectedly large" + tokens = shlex.split(command) + assert tokens[:2] == ["jq", "-n"], "the jq invocation shape changed" + arguments = {} + cursor = 2 + while cursor < len(tokens) and tokens[cursor] == "--arg": + name, variable = tokens[cursor + 1:cursor + 3] + assert variable.startswith("$") and variable[1:].isidentifier() + arguments[name] = variable[1:] + cursor += 3 + assert cursor + 1 < len(tokens), "the jq filter or output file is missing" + jq_filter = tokens[cursor] + output = tokens[cursor + 1] + assert output.startswith(">") and len(output) > 1 values = { name: ( {"runtime": "images.runtime", "file-service": "images.fileService", @@ -75,7 +98,7 @@ def record_image_identity(component: str, directory: pathlib.Path) -> None: for name, variable in arguments.items() } record = {} - for key, expression in re.findall(r"(\w+):(\$\w+|\([^)]*\))", match.group(2)): + for key, expression in re.findall(r"(\w+):(\$\w+|\([^)]*\))", jq_filter): if expression.startswith("$"): record[key] = values[expression[1:]] else: @@ -83,7 +106,7 @@ def record_image_identity(component: str, directory: pathlib.Path) -> None: values[part[1:]] if part.startswith("$") else json.loads(part) for part in re.findall(r'\$\w+|"[^"]*"', expression) ) - target = match.group(3).replace("${COMPONENT}", component) + target = output[1:].replace("${COMPONENT}", component) (directory / target).write_text(json.dumps(record), encoding="utf-8")