From 0dc19a4618e72a894476a42919ebd337e8a2e3a3 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:00:34 +0000 Subject: [PATCH 1/2] chore(sandbox): remove the two spent probe scripts; add the review page snapshot-probe.ts created swapfiles to measure what a snapshot keeps, and drive-fork-probe.ts measured drive forks on throwaway drives. Both results are in plans/20261116-thin-sandbox-findings.md, boxes have no swap since #9924, and the real-sandbox drive pass now times a fork from a golden with content. Nothing imports either script; the workflow option for the fork probe goes with it. --- .github/workflows/sandbox.yml | 26 -- packages/sandbox/src/drive-fork-probe.ts | 184 ------------- packages/sandbox/src/snapshot-probe.ts | 119 -------- plans/20261118-thin-sandbox-review.html | 328 +++++++++++++++++++++++ 4 files changed, 328 insertions(+), 329 deletions(-) delete mode 100644 packages/sandbox/src/drive-fork-probe.ts delete mode 100644 packages/sandbox/src/snapshot-probe.ts create mode 100644 plans/20261118-thin-sandbox-review.html diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 0e77ecaa20e..31d4d56969c 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -23,7 +23,6 @@ on: options: - real-sandbox - production-release - - drive-fork-probe - inventory default: real-sandbox @@ -149,31 +148,6 @@ jobs: CDN_URL: ${{ secrets.CDN_URL }} run: bun run src/real-sandbox.ts - drive-fork-probe: - name: Drive fork probe - if: github.event_name == 'workflow_dispatch' && inputs.run == 'drive-fork-probe' - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Check out code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - with: - bun-version-file: .bun-version - - - name: Install dependencies - run: bun install --frozen --ignore-scripts - - - name: Measure drive forks on throwaway drives - working-directory: packages/sandbox - env: - VERCEL_SANDBOX_TOKEN: ${{ secrets.VERCEL_SANDBOX_TOKEN }} - VERCEL_SANDBOX_TEAM_ID: ${{ secrets.VERCEL_SANDBOX_TEAM_ID }} - VERCEL_SANDBOX_PROJECT_ID: ${{ secrets.VERCEL_SANDBOX_PROJECT_ID }} - run: bun run src/drive-fork-probe.ts - inventory: name: Snapshot inventory if: github.event_name == 'workflow_dispatch' && inputs.run == 'inventory' diff --git a/packages/sandbox/src/drive-fork-probe.ts b/packages/sandbox/src/drive-fork-probe.ts deleted file mode 100644 index 0b801e591c8..00000000000 --- a/packages/sandbox/src/drive-fork-probe.ts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Measures what a Vercel drive fork costs, on throwaway `spike-fork-*` - * drives and sandboxes that it deletes: how long a fork of a realistic - * workspace drive takes, whether the fork is independent of its parent, - * and how cold reads from a fresh fork compare with the parent. - * - * bun run src/drive-fork-probe.ts needs VERCEL_SANDBOX_* credentials - */ -import { Drive, Sandbox } from "@vercel/sandbox"; - -const credentials = { - token: process.env.VERCEL_SANDBOX_TOKEN ?? "", - teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", - projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", -}; -if (!credentials.token || credentials.token.startsWith("fake-")) { - console.error("VERCEL_SANDBOX_TOKEN, _TEAM_ID and _PROJECT_ID are required"); - process.exit(64); -} - -const REGION = "sfo1"; -const MOUNT = "/mnt/drive"; -const REPO = "https://github.com/ParadevAI/Paradev.git"; -const run = Date.now().toString(36); -const started = Date.now(); -const log = (line: string) => - console.log( - `${((Date.now() - started) / 1000).toFixed(0).padStart(4)}s ${line}`, - ); -const drives: string[] = []; -const boxes: string[] = []; - -async function timed(label: string, work: () => Promise): Promise { - const from = Date.now(); - const result = await work(); - log(`${label}: ${((Date.now() - from) / 1000).toFixed(1)} s`); - return result; -} - -async function box(name: string, drive: Drive): Promise { - boxes.push(name); - return Sandbox.create({ - ...credentials, - name, - region: REGION, - persistent: false, - resources: { vcpus: 4 }, - timeout: 20 * 60_000, - mounts: { [MOUNT]: drive }, - }); -} - -async function sh(sandbox: Sandbox, script: string): Promise { - const result = await sandbox.runCommand({ - cmd: "bash", - args: ["-c", `exec 2>&1; ${script}`], - sudo: true, - }); - const out = (await result.output("both")).trim(); - if (result.exitCode !== 0) log(` exit ${result.exitCode}`); - return out; -} - -async function release(sandbox: Sandbox, drive: string): Promise { - await sandbox.stop(); - for (let i = 0; i < 120; i++) { - const now = await Sandbox.get({ - ...credentials, - name: sandbox.name, - resume: false, - }); - const held = (await Drive.get({ ...credentials, name: drive })) - .currentSandboxName; - if (now.status === "stopped" && !held) return; - await new Promise((resolve) => setTimeout(resolve, 2_000)); - } - throw new Error(`${drive} still attached after ${sandbox.name} stopped`); -} - -const READ_ALL = `export HOME=/root; cd ${MOUNT}/repo && ms(){ date +%s%3N; }; s=$(ms); git status --short | wc -l; echo "git status $(( $(ms)-s )) ms"; s=$(ms); find node_modules -type f -print0 | xargs -0 -P 8 -n 500 cat > /dev/null; echo "read all of node_modules $(( $(ms)-s )) ms"`; - -try { - const baseName = `spike-fork-base-${run}`; - drives.push(baseName); - const base = await Drive.getOrCreate({ - ...credentials, - name: baseName, - region: REGION, - maxSize: 40 * 1024 ** 3, - }); - - const writer = await box(`spike-fork-writer-${run}`, base); - log( - await sh( - writer, - `chmod 777 ${MOUNT} && cd ${MOUNT} && export HOME=/root && git config --global --add safe.directory '*' && git clone -q --depth 1 ${REPO} repo && cd repo && npm i -g bun@$(cat .bun-version) >/dev/null 2>&1; BUN_INSTALL_CACHE_DIR=${MOUNT}/store bun install --ignore-scripts 2>&1 | tail -2; head -c 6G /dev/urandom > ${MOUNT}/big.bin; echo base > ${MOUNT}/marker; sync; du -sh ${MOUNT} | cut -f1; find node_modules -type f | wc -l`, - ), - ); - await timed("writer stop and drive release", () => release(writer, baseName)); - - const forkName = `spike-fork-a-${run}`; - drives.push(forkName); - const fork = await timed("FORK of the populated drive", () => - base.fork({ name: forkName }), - ); - log( - `fork: parentDriveId ${fork.parentDriveId === base.driveId ? "is the base" : fork.parentDriveId}, region ${fork.region}, maxSize ${fork.maxSize}`, - ); - - const secondName = `spike-fork-b-${run}`; - drives.push(secondName); - await timed("second FORK of the same drive", () => - base.fork({ name: secondName }), - ); - - const reader = await timed("sandbox create with the fork mounted", () => - box(`spike-fork-reader-${run}`, fork), - ); - log( - `fork, first read:\n${await sh(reader, `git config --global --add safe.directory '*'; ${READ_ALL}`)}`, - ); - log(`fork, second read:\n${await sh(reader, READ_ALL)}`); - log( - await sh( - reader, - `cat ${MOUNT}/marker; echo fork > ${MOUNT}/marker; head -c 1G /dev/urandom > ${MOUNT}/only-in-fork.bin; sync; df -h ${MOUNT} | tail -1`, - ), - ); - await timed("reader stop and fork release", () => release(reader, forkName)); - - const check = await box(`spike-fork-check-${run}`, base); - log( - `base after the fork was written: marker=${await sh(check, `cat ${MOUNT}/marker; ls ${MOUNT} | tr '\\n' ' '`)}`, - ); - log( - `base, read after forks exist:\n${await sh(check, `git config --global --add safe.directory '*'; ${READ_ALL}`)}`, - ); - await release(check, baseName); - - const grandName = `spike-fork-c-${run}`; - drives.push(grandName); - await timed("FORK of a fork (1 GB written since)", () => - fork.fork({ name: grandName }), - ); - - await timed("delete the base while forks exist", async () => { - await base.delete(); - drives.splice(drives.indexOf(baseName), 1); - }).catch((error) => - log(`delete base refused: ${String(error).slice(0, 200)}`), - ); - const orphan = await box( - `spike-fork-orphan-${run}`, - await Drive.get({ ...credentials, name: secondName }), - ); - log( - `fork b after its parent was deleted:\n${await sh(orphan, `git config --global --add safe.directory '*'; cat ${MOUNT}/marker; ${READ_ALL}`)}`, - ); - await release(orphan, secondName); -} catch (error) { - log(`FAILED: ${String(error).slice(0, 400)}`); - process.exitCode = 1; -} finally { - for (const name of boxes) { - const sandbox = await Sandbox.get({ - ...credentials, - name, - resume: false, - }).catch(() => null); - if (!sandbox) continue; - if (sandbox.status === "running") await sandbox.stop().catch(() => {}); - await sandbox.delete().catch(() => {}); - } - await new Promise((resolve) => setTimeout(resolve, 15_000)); - for (const name of drives.reverse()) { - const drive = await Drive.get({ ...credentials, name }).catch(() => null); - await drive - ?.delete() - .catch((error) => - log(`left behind: ${name}: ${String(error).slice(0, 120)}`), - ); - } - log("cleaned up"); -} diff --git a/packages/sandbox/src/snapshot-probe.ts b/packages/sandbox/src/snapshot-probe.ts deleted file mode 100644 index 0806c0c2693..00000000000 --- a/packages/sandbox/src/snapshot-probe.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Probes what a stop snapshots, on one throwaway sandbox: - * - does a written file grow the snapshot, and does deleting it shrink it - * - does a swapfile count, active or deleted - * - do processes get SIGTERM before the snapshot, and can they still write - * - * bun run packages/sandbox/src/snapshot-probe.ts - */ -import { Sandbox, Snapshot } from "@vercel/sandbox"; - -const credentials = { - token: process.env.VERCEL_SANDBOX_TOKEN ?? "", - teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", - projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", -}; -const name = `spike-snapshot-probe-${Date.now().toString(36)}`; -const started = Date.now(); -const log = (line: string) => - console.log( - `${((Date.now() - started) / 1000).toFixed(0).padStart(4)}s ${line}`, - ); -const mb = (bytes: number) => `${(bytes / 1024 / 1024).toFixed(0)} MB`; - -async function sh(script: string): Promise { - const sandbox = await Sandbox.get({ ...credentials, name, resume: true }); - const result = await sandbox.runCommand({ - cmd: "bash", - args: ["-c", script], - sudo: true, - }); - const out = (await result.output("both")).trim(); - if (result.exitCode !== 0) - log(` exit ${result.exitCode}: ${out.slice(0, 300)}`); - return out; -} - -async function stopAndMeasure(label: string): Promise { - const sandbox = await Sandbox.get({ ...credentials, name, resume: false }); - const before = sandbox.currentSnapshotId; - const t = Date.now(); - await sandbox.stop(); - for (let i = 0; i < 240; i++) { - const now = await Sandbox.get({ ...credentials, name, resume: false }); - if ( - now.status === "stopped" && - now.currentSnapshotId && - now.currentSnapshotId !== before - ) { - const snapshot = await Snapshot.get({ - ...credentials, - snapshotId: now.currentSnapshotId, - }); - log( - `${label}: snapshot ${mb(snapshot.sizeBytes)} (stop took ${((Date.now() - t) / 1000).toFixed(0)}s)`, - ); - return snapshot.sizeBytes; - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - throw new Error(`${label}: no new snapshot`); -} - -try { - await Sandbox.create({ - ...credentials, - name, - resources: { vcpus: 2 }, - timeout: 20 * 60 * 1000, - persistent: true, - keepLastSnapshots: { count: 1 }, - }); - log(`created ${name}`); - log( - await sh( - "findmnt -no SOURCE,FSTYPE,OPTIONS /; lsblk -D -o NAME,DISC-GRAN,DISC-MAX | head -4; df -h / | tail -1", - ), - ); - const base = await stopAndMeasure("baseline"); - - await sh("head -c 3G /dev/urandom > /var/big.bin; sync"); - const written = await stopAndMeasure("3 GB random file written"); - - await sh("rm -f /var/big.bin; sync; fstrim -v / 2>&1 || true"); - const deleted = await stopAndMeasure("file deleted"); - - log( - await sh( - "mkdir -p /var/swap && fallocate -l 2G /var/swap/a && chmod 600 /var/swap/a && mkswap /var/swap/a >/dev/null && swapon /var/swap/a && swapon --show", - ), - ); - const swapOn = await stopAndMeasure("2 GB swapfile active, unused"); - - // A process that reacts to SIGTERM: can a box clean up at stop? - await sh(`rm -f /var/term.log; nohup bash -c 'trap "date +%s%3N >> /var/term.log; swapoff /var/swap/a; rm -f /var/swap/a; date +%s%3N >> /var/term.log; sync; exit 0" TERM; while true; do sleep 1; done' >/dev/null 2>&1 & -sleep 1; echo started`); - const afterTrap = await stopAndMeasure( - "stop with a SIGTERM trap that removes swap", - ); - log( - `term.log after resume: [${await sh("cat /var/term.log 2>/dev/null; ls -la /var/swap 2>/dev/null")}]`, - ); - - console.log( - JSON.stringify({ base, written, deleted, swapOn, afterTrap }, null, 2), - ); -} finally { - const sandbox = await Sandbox.get({ - ...credentials, - name, - resume: false, - }).catch(() => null); - if (sandbox) { - if (sandbox.status === "running") await sandbox.stop().catch(() => {}); - await sandbox - .delete({ deleteOrphanSnapshots: true }) - .catch((error) => log(`delete failed: ${error}`)); - log(`deleted ${name}`); - } -} diff --git a/plans/20261118-thin-sandbox-review.html b/plans/20261118-thin-sandbox-review.html new file mode 100644 index 00000000000..d7446e4729b --- /dev/null +++ b/plans/20261118-thin-sandbox-review.html @@ -0,0 +1,328 @@ + + + + + +Thin sandbox: PRs for review + + + +
+ +

Thin sandbox: PRs for review

+

Written 18 November 2026. Four pieces of work, each with what it does, pseudocode, what a customer sees, the evidence, and the decisions that are yours. Nothing here is merged.

+ +

Summary

+
+ + + + + + + + +
WorkPRsStateWhat it buysYour decisions
Remove swap#9924Ready for reviewNo swap in snapshots (up to 6 to 16 GB per workspace that hit memory pressure). No long stalls under a fast allocator.1
Agents only from the drive#9929, then #9911A: ready for review B: after A is released0.79 GB off the image. A PostHog outage can no longer affect agents.3
Environment builds on Vercel Workflow#9906, on #9282Waiting on your decisionsBuilds can run 30 minutes, not about 3. The build log updates while it runs.3
Workspace drive#9915, #9847, model #9914Not ready: model verdict outstandingRoot snapshot 2.76 GB against 7.86 to 8.02 GB for the same work.2, later
+
+ +

All decisions in one place

+
    +
  1. Swap: ship no swap with the faster memguard?
  2. +
  3. Agents: when the agents drive cannot be mounted, open the box without agents, or fail the open?
  4. +
  5. Agents: should workspaces of archived environments move to the current bundle?
  6. +
  7. Agents: publish drives in all 19 regions on every release, or only regions that have an environment?
  8. +
  9. Builds: bump the Vercel CLI for staging and production from 50.22.1 to 50.31.0?
  10. +
  11. Builds: is 30 minutes the right build limit?
  12. +
  13. Builds: who rebases #9282, and when?
  14. +
  15. Drive (later): which internal environment gets it first, and what happens to a drive when its workspace expires after 30 days?
  16. +
+ + +

1. Remove swap Ready for review

+

PR: #9924, draft. 5 files, +86 −87. All 18 checks pass.

+ +

What it does

+

Cloud boxes stop using swap. Removing swap alone was not safe: host-service stalled for up to 175 seconds. So the memory guard that kills runaway processes now checks four times as often and acts one step ahead.

+ +

Pseudocode

+
every 0.25 s:                              # was every 1 s
+    read available memory                  # no process spawned
+    drop = how much it fell since the last check
+    if available - drop < 8% of the box:
+        kill the largest ordinary process  # never host-service, terminals or agents
+        wait until it has really exited    # was a fixed 3 s pause
+        check again at once
+ +

What a customer sees

+
    +
  • A job bigger than the box is killed early. With swap it often finished slowly. A parallel build on a 4 GB box loses 3 of its 4 workers. A monorepo typecheck of about 8 GB is killed on the 4 GB size.
  • +
  • Within about 5 seconds the desktop shows an out-of-memory banner that names the process and offers a resize to the next size. The terminal shows Killed.
  • +
  • Host-service, terminals and agents are never the ones killed.
  • +
+ +

Evidence

+

Measured on a real 2 vCPU / 4 GB Vercel box with its real memory limit, no swap, memguard from this PR:

+
+ + + + + + + + +
CaseLongest host-service gapReapedKernel killsProtected processes killed
One steady allocator256 ms1, at 22.5 s0none
Parallel build, 4 workers256 ms3 of 4, first at 5.3 s0none
Page cache plus allocator256 ms1, at 28.8 s0none
Fast allocator, three runs254 ms each1 each, at 0.9 s0none
+
+

The worker's simulated runs at 4 GB and 8 GB gave the same picture, with a longest gap of 314 ms. On main, with swap and the 1-second check, the fast allocator caused stalls of 6 to 94 seconds in 6 of 7 simulated runs.

+

Not measured: snapshot size before and after on real Vercel. Swap stayed empty in every run, so the saving follows, but it has not been weighed.

+ +

Risk and rollback

+
    +
  • The main risk is the product trade above: oversized jobs die where they used to finish.
  • +
  • Rollback is a revert. No boot step re-runs. Snapshots that already hold swap keep it either way.
  • +
+ +

Your decision

+
+

Ship no swap with this loop?

+

Recommendation: yes. You settled on removing swap after Vercel's advice. The stall that made it unsafe is fixed and proven on a real small box, and main is worse under a fast allocator.

+
+

Not blocking, worth scheduling: a killed Node process shows in the banner as "MainThread", and on the largest size the banner offers no next step.

+ + +

2. Agents only from the drive A: ready for review B: after A is released

+

PRs: #9929 (release side) and #9911 (box side, stacked on it). Both drafts.

+ +

What it does

+

Claude, Codex and Pdev exist twice today: on every box's disk and on the shared read-only agents drive. This removes the disk copy. It is split in two so nothing depends on deploy timing.

+
    +
  • A, #9929 changes only the release. It changes nothing on any box and can soak alone.
  • +
  • B, #9911 removes the disk copy and the flag. It merges only after a release with A has succeeded.
  • +
+ +

Pseudocode

+
# A: the release
+publish the agents drive in all 19 regions
+fail the release if a region with an environment has no drive,
+    or a drive without claude, codex or pdev
+move every environment row to the new bundle, archived ones too
+keep the drives of the 3 newest bundles          # so a rollback has something to mount
+only the production release prunes drives
+
+# B: create or wake a box
+always try to mount the region's agents drive    # no flag
+retry on a transient error
+still no drive -> open the box anyway, report it to us
+boot:
+    link agents from the drive
+    only after a successful link: remove the copies on the box's disk
+agent launched with no drive:
+    "Agents are unavailable on this box; reopen the workspace"
+ +

What a customer sees

+
    +
  • When it works: nothing. Same agents, same paths.
  • +
  • When the drive cannot be mounted: the box opens with terminals, files and desktop working, and launching an agent gives the message above.
  • +
  • Workspaces of an archived environment move to the current bundle on their next wake. Three such workspaces exist in production.
  • +
+ +

Evidence

+
    +
  • Real Vercel, on an earlier head of B: a drive was published for the branch's bundle, all three agents ran from it, and they were linked again after a stop and wake.
  • +
  • Local image: 4.89 GB to 4.10 GB.
  • +
  • The agents drive has been on for every organization since 17 November, 23:38 UTC. Two customer organizations were seen picking it up, one on create and one on wake.
  • +
+

Not verified: the current heads of A and B on real Vercel (A's run is in progress), how long publishing 19 regions takes, and a rollback rehearsal.

+ +

Risk and rollback

+
    +
  • An independent review of the first version said "do not merge". It found that rollback did not restore agents, that an incomplete drive could ship, and that archived environments' workspaces could never wake. A and B are the response. They have not been re-reviewed.
  • +
  • A release can now fail on a drive problem. Boxes then keep the previous bundle for one more cycle.
  • +
  • Rollback of B: revert it. Boxes that already dropped their disk copy get it back, because the install marker is now removed with it. The kept drives cover the previous bundles.
  • +
  • B conflicts with the workspace drive (#9847) in six files. Whichever lands second rebases.
  • +
+ +

Your decisions

+
+

No drive: open without agents, or fail the open?

+

Recommendation: open without agents, which is what B implements. A box with working terminals and files is better than no box, and it means a drive problem can never mark a workspace failed.

+
+
+

Should workspaces of archived environments move to the current bundle?

+

Recommendation: yes. Otherwise their drive is pruned and they lose agents.

+
+
+

Drives in all 19 regions on every release, or only regions with an environment?

+

Recommendation: all 19, since an environment can be created in any region before the next release. The cost is a longer release and more ways for the nightly to fail. The first release on A will give the real publish time.

+
+ + +

3. Environment builds on Vercel Workflow Waiting on your decisions

+

PR: #9906, draft, stacked on your #9282.

+ +

What it does

+

A build is one serverless function today, with a 285-second budget. This makes it a Vercel Workflow run: a series of short steps, each retried on failure, with no compute used while it waits.

+ +

Pseudocode

+
environmentBuildWorkflow(buildId):
+    deadline = now + 30 min
+    create the build box                   # found by name on retry, never a second box
+    for command in [checkout, install]:
+        start it detached on the box       # runs once per command
+        until it exits:
+            save the last 40 lines to the build row
+            sleep 5 s
+    stop the box, poll until the snapshot exists
+    activate the new golden                # no second swap on retry
+    on any failure: delete the box, mark the build failed with the reason
+ +

What a customer sees

+
    +
  • A build can take up to 30 minutes. Today an install over about 3 minutes fails.
  • +
  • Same build IDs, statuses and table columns. No tRPC output changes.
  • +
  • The stored log updates every 5 seconds during a build. Nothing in the UI reads it yet.
  • +
+ +

Evidence

+
    +
  • A local run of the API app with Workflow's local runner and a fake build box: a successful build and a cancelled build both ran end to end.
  • +
  • Tests with no real waiting: slow install, failed install, a step that fails then succeeds on retry, the limit, cleanup.
  • +
  • Preview deploys of API, Web, Docs and Marketing pass on the newer CLI.
  • +
  • Cost at Vercel's listed price: about $0.004 for a typical build, $0.04 for a 30-minute one.
  • +
+

Not verified: a real build on Vercel's workflow runtime with a real sandbox. A run stays pinned to the deployment that started it; one whose deployment is deleted is failed by the stale check after 40 minutes.

+ +

Your decisions

+
+

Bump the Vercel CLI for staging and production from 50.22.1 to 50.31.0?

+

Workflow hard-codes a setting our pinned CLI rejects. 50.31.0 is the lowest version that accepts it, and it is the same major. It is set for previews in this draft only.

+

Recommendation: yes, as its own small PR, watched on staging first.

+
+
+

Is 30 minutes the right build limit?

+

Recommendation: yes to start. It is one constant.

+
+
+

Who rebases #9282 onto main, and when?

+

Its migration number is stale, which is why the preview database step fails. It is your in-progress work, so I have not touched it.

+

Recommendation: I can start a worker on the rebase if you want it off your plate.

+
+

Also needed before it runs anywhere: "Enable access to System Environment Variables" on the API's Vercel project. Nobody has checked whether it is on.

+ + +

4. Workspace drive Not ready: model verdict outstanding

+

PRs: #9915 (two nullable columns, lands first), #9847 (the change), #9914 (the TLA+ model). All drafts.

+ +

What it does

+

A workspace's work moves onto a Vercel Drive mounted beside the box, so the box's own snapshot stays small however much work is done. Everything still persists.

+ +

Pseudocode

+
create a workspace, drive mode on:
+    fork the environment's golden drive        # about 1 s, copy-on-write
+    mount it; first boot lays it out and records the owner on the drive
+boot:
+    drive expected but not mounted, or not this workspace's -> refuse to boot
+    bind /workspace, home caches, docker data and swap onto the drive
+    fresh fork -> fetch and create the branch; existing work -> never replaced
+wake:
+    under one lock: wait for the drive to be released, mount, resume
+delete:
+    retire the box and drive names in the same write
+    cleanup after the grace period, never of anything a live row names
+undo delete:
+    resumes the same box and drive if they still exist
+ +

The rules it must keep

+

These are the model's rules. The last column is the model author's result at the previous head. The fix worker reports all ten hold at the current head, but it edited the model to get there, so the author is checking that independently.

+
+ + + + + + + + + + + + + + +
RuleHarm if brokenIndependently verified
Nothing is lostCommits or changes disappearfailed, fix claimed
Never empty shown as workUser sees an empty workspace as their projectholds
One writer per driveDrive reads as corrupt or emptyfailed, fix claimed
Cleanup never deletes a live workspaceA workspace in use loses its filesfailed, fix claimed
Only creating a workspace gives it a driveOld workspaces break when drive mode turns onholds
No drive, no changeThe flag being off is not safeholds
Never stuck for everWork is safe but cannot be openedfailed, fix claimed
A new workspace starts on fresh codeStale code that looks like the user's branchfailed, fix claimed
A passing glitch is not a failureA workspace that would come up shows as brokenholds
No drive left behind for everStorage billed for everholds only if the sweep is run by hand
+
+ +

Evidence

+

Real Vercel, same work on both boxes (clone, bun install, 4 GB written), four runs:

+
+ + + + + + + + +
Plain boxDrive box
Root snapshot after stop7.86 to 8.02 GB2.58 to 2.79 GB
Create to host-service ready4.7 to 5.3 s3.7 to 6.4 s
bun install70 to 74 s74 to 83 s
Wake, total6.8 to 9.4 s5.4 to 6.8 s
+
+
    +
  • Kept across stop and wake on both boxes: an uncommitted file, a local commit, a home file, a root-disk file, a Docker image.
  • +
  • After an unclean stop: the box boots, host.db passes its integrity check, the checkout is intact.
  • +
  • A workspace created from a drive-mode golden boots on its own new branch with dependencies present.
  • +
+

Not measured: a cold wake, which is where the size should pay off most and which cannot be forced from the SDK. The wake numbers above were probably all warm, so do not read a wake improvement from them. Also unmeasured: whether Vercel restarts a stopped box on certain calls, which decides how severe one lifecycle gap is.

+ +

Why it is not ready

+

The benefit is solid. The lifecycle has had four rounds of fixes, and in three of them a fix introduced a new bug that only the model caught. Two things must both be true on the same commit before I recommend enabling it for even one internal environment:

+
    +
  1. The model author confirms every rule holds, having re-derived the model from the code.
  2. +
  3. The real-Vercel run passes. It does at the current head.
  4. +
+

Still unbuilt: migrating existing workspaces, scheduling the orphan sweep, and merging with the agents work in section 2.

+ +

Your decisions, when it is ready

+
+

Which internal environment gets drive mode first? New workspaces only. There is a kill switch.

+
+
+

What happens to a drive when its workspace's snapshot expires after 30 days? Today the snapshot expires and the drive does not, so an abandoned workspace keeps billing for a drive nothing can resume.

+
+ +

Already shipped in this effort

+
+ + + + + + + + + + +
PRWhatMeasured effect
#9905Host-service starts before the other boot stepsA fresh box is ready in about 5 s on real Vercel, was about 30 s
#9896, #9912, #9916Watcher stops watching ignored directories created laterReal bun install: peak 506 MB, was 1,350 MB; 244 MB six minutes later, was about 1,444 MB
#9866OpenCode and Pi on the agents driveVerified on a fresh box
#9865A failed boot step skips only its dependentsA Chrome failure no longer removes agents
#9869agent_launched telemetryReports after the next desktop and box releases
FlagAgents drive on for every organizationTwo customer organizations confirmed
+
+ +
+ + From 92566f253fde94c5246e56daa3979450eacc806f Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:11:44 +0000 Subject: [PATCH 2/2] docs(plans): the review page says swap removal shipped, and when an agent can be reaped --- plans/20261118-thin-sandbox-review.html | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plans/20261118-thin-sandbox-review.html b/plans/20261118-thin-sandbox-review.html index d7446e4729b..1d2d1d20e24 100644 --- a/plans/20261118-thin-sandbox-review.html +++ b/plans/20261118-thin-sandbox-review.html @@ -30,14 +30,14 @@

Thin sandbox: PRs for review

-

Written 18 November 2026. Four pieces of work, each with what it does, pseudocode, what a customer sees, the evidence, and the decisions that are yours. Nothing here is merged.

+

Written 18 November 2026. Four pieces of work, each with what it does, pseudocode, what a customer sees, the evidence, and the decisions that are yours. Swap removal has shipped; the other three are not merged.

Summary

- + @@ -47,7 +47,7 @@

Summary

All decisions in one place

    -
  1. Swap: ship no swap with the faster memguard?
  2. +
  3. Swap: ship no swap with the faster memguard? Decided: yes, shipped.
  4. Agents: when the agents drive cannot be mounted, open the box without agents, or fail the open?
  5. Agents: should workspaces of archived environments move to the current bundle?
  6. Agents: publish drives in all 19 regions on every release, or only regions that have an environment?
  7. @@ -58,8 +58,8 @@

    All decisions in one place

-

1. Remove swap Ready for review

-

PR: #9924, draft. 5 files, +86 −87. All 18 checks pass.

+

1. Remove swap Shipped 18 November

+

PR: #9924, merged. 5 files, +86 −87. The production release that carries it succeeded, and boxes pick it up on their next wake.

What it does

Cloud boxes stop using swap. Removing swap alone was not safe: host-service stalled for up to 175 seconds. So the memory guard that kills runaway processes now checks four times as often and acts one step ahead.

@@ -69,7 +69,8 @@

Pseudocode

read available memory # no process spawned drop = how much it fell since the last check if available - drop < 8% of the box: - kill the largest ordinary process # never host-service, terminals or agents + kill the largest ordinary process # agents only if no ordinary process is left; + # host-service and terminals never wait until it has really exited # was a fixed 3 s pause check again at once @@ -77,7 +78,7 @@

What a customer sees

  • A job bigger than the box is killed early. With swap it often finished slowly. A parallel build on a 4 GB box loses 3 of its 4 workers. A monorepo typecheck of about 8 GB is killed on the 4 GB size.
  • Within about 5 seconds the desktop shows an out-of-memory banner that names the process and offers a resize to the next size. The terminal shows Killed.
  • -
  • Host-service, terminals and agents are never the ones killed.
  • +
  • Host-service and terminals are never the ones killed. An agent is killed only when no ordinary process is left to kill. In every measured run the victim was an ordinary process.

Evidence

@@ -102,10 +103,9 @@

Risk and rollback

  • Rollback is a revert. No boot step re-runs. Snapshots that already hold swap keep it either way.
  • -

    Your decision

    +

    Decision

    -

    Ship no swap with this loop?

    -

    Recommendation: yes. You settled on removing swap after Vercel's advice. The stall that made it unsafe is fixed and proven on a real small box, and main is worse under a fast allocator.

    +

    Ship no swap with this loop? Decided: yes. Merged and released on 18 November.

    Not blocking, worth scheduling: a killed Node process shows in the banner as "MainThread", and on the largest size the banner offers no next step.

    WorkPRsStateWhat it buysYour decisions
    Remove swap#9924Ready for reviewNo swap in snapshots (up to 6 to 16 GB per workspace that hit memory pressure). No long stalls under a fast allocator.1
    Remove swap#9924Shipped 18 NovemberNo swap in snapshots (up to 6 to 16 GB per workspace that hit memory pressure). No long stalls under a fast allocator.Decided: ship
    Agents only from the drive#9929, then #9911A: ready for review B: after A is released0.79 GB off the image. A PostHog outage can no longer affect agents.3
    Environment builds on Vercel Workflow#9906, on #9282Waiting on your decisionsBuilds can run 30 minutes, not about 3. The build log updates while it runs.3
    Workspace drive#9915, #9847, model #9914Not ready: model verdict outstandingRoot snapshot 2.76 GB against 7.86 to 8.02 GB for the same work.2, later