From 4504279c1cdb8cc4fcc5fb3c380b9406726c4ba2 Mon Sep 17 00:00:00 2001 From: dotcomnerd <70242273+dotcomnerd@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:20:03 -0700 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=97=82=EF=B8=8F=20move=20docker=20com?= =?UTF-8?q?pose=20under=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.cap.yml => infra/docker-compose.cap.yml | 0 docker-compose.crowdsec.yml => infra/docker-compose.crowdsec.yml | 0 .../docker-compose.ffs.local.yml | 0 docker-compose.ffs.yml => infra/docker-compose.ffs.yml | 0 docker-compose.imgproxy.yml => infra/docker-compose.imgproxy.yml | 0 .../docker-compose.local-stack.yml | 0 docker-compose.scrape.yml => infra/docker-compose.scrape.yml | 0 docker-compose.yml => infra/docker-compose.yml | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename docker-compose.cap.yml => infra/docker-compose.cap.yml (100%) rename docker-compose.crowdsec.yml => infra/docker-compose.crowdsec.yml (100%) rename docker-compose.ffs.local.yml => infra/docker-compose.ffs.local.yml (100%) rename docker-compose.ffs.yml => infra/docker-compose.ffs.yml (100%) rename docker-compose.imgproxy.yml => infra/docker-compose.imgproxy.yml (100%) rename docker-compose.local-stack.yml => infra/docker-compose.local-stack.yml (100%) rename docker-compose.scrape.yml => infra/docker-compose.scrape.yml (100%) rename docker-compose.yml => infra/docker-compose.yml (100%) diff --git a/docker-compose.cap.yml b/infra/docker-compose.cap.yml similarity index 100% rename from docker-compose.cap.yml rename to infra/docker-compose.cap.yml diff --git a/docker-compose.crowdsec.yml b/infra/docker-compose.crowdsec.yml similarity index 100% rename from docker-compose.crowdsec.yml rename to infra/docker-compose.crowdsec.yml diff --git a/docker-compose.ffs.local.yml b/infra/docker-compose.ffs.local.yml similarity index 100% rename from docker-compose.ffs.local.yml rename to infra/docker-compose.ffs.local.yml diff --git a/docker-compose.ffs.yml b/infra/docker-compose.ffs.yml similarity index 100% rename from docker-compose.ffs.yml rename to infra/docker-compose.ffs.yml diff --git a/docker-compose.imgproxy.yml b/infra/docker-compose.imgproxy.yml similarity index 100% rename from docker-compose.imgproxy.yml rename to infra/docker-compose.imgproxy.yml diff --git a/docker-compose.local-stack.yml b/infra/docker-compose.local-stack.yml similarity index 100% rename from docker-compose.local-stack.yml rename to infra/docker-compose.local-stack.yml diff --git a/docker-compose.scrape.yml b/infra/docker-compose.scrape.yml similarity index 100% rename from docker-compose.scrape.yml rename to infra/docker-compose.scrape.yml diff --git a/docker-compose.yml b/infra/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to infra/docker-compose.yml From 18a2291d28acf1aa6f0c3dfbee61d88533692b73 Mon Sep 17 00:00:00 2001 From: dotcomnerd <70242273+dotcomnerd@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:23:29 -0700 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20harden=20runtime=20?= =?UTF-8?q?and=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 21 +- README.md | 194 ++++++++-------- apps/web/app/api/tmdb/proxy/route.ts | 268 ++++++++++++++++++++++ apps/web/app/login/auth-shell.tsx | 5 +- apps/web/auth.ts | 2 +- apps/web/drizzle.config.ts | 14 +- apps/web/package.json | 4 + infra/docker-compose.local-stack.yml | 12 +- infra/docker-compose.scrape.yml | 13 +- scripts/db-migrate-if-needed.sh | 72 ++++-- scripts/deploy.sh | 19 ++ scripts/gluetun/docker-compose.local.yml | 1 + scripts/gluetun/docker-compose.yml | 10 + scripts/infra-health.sh | 139 +++++++++++ scripts/install-prod-watchdogs.sh | 39 ++++ scripts/local-compose.sh | 2 +- scripts/nginx-cache.conf | 30 +++ scripts/nyumatflix-infra-watchdog.service | 14 ++ scripts/nyumatflix-infra-watchdog.sh | 43 ++++ scripts/nyumatflix-infra-watchdog.timer | 11 + scripts/reconcile-cap.sh | 4 +- scripts/reconcile-crowdsec.sh | 4 +- scripts/reconcile-prod-infra.sh | 34 ++- scripts/setup-vpn.sh | 12 +- scripts/sync-prod-env.sh | 25 +- 25 files changed, 824 insertions(+), 168 deletions(-) create mode 100644 apps/web/app/api/tmdb/proxy/route.ts create mode 100644 scripts/infra-health.sh create mode 100644 scripts/install-prod-watchdogs.sh create mode 100644 scripts/nginx-cache.conf create mode 100644 scripts/nyumatflix-infra-watchdog.service create mode 100644 scripts/nyumatflix-infra-watchdog.sh create mode 100644 scripts/nyumatflix-infra-watchdog.timer diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa9c7731..d42a0d6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -172,6 +172,19 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install --ignore-scripts + + - name: Sync production database + env: + PROD_DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }} + run: ./scripts/db-migrate-if-needed.sh + - name: Set up SSH key run: | install -m 700 -d ~/.ssh @@ -181,12 +194,12 @@ jobs: - name: Sync deploy files run: | - ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p '$DEPLOY_PATH/scripts'" + ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p '$DEPLOY_PATH/infra' '$DEPLOY_PATH/scripts'" rsync -avz -e "ssh -p $DEPLOY_PORT" \ - docker-compose.scrape.yml docker-compose.ffs.yml docker-compose.imgproxy.yml \ - "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + infra/docker-compose.scrape.yml infra/docker-compose.ffs.yml infra/docker-compose.imgproxy.yml \ + "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/infra/" rsync -avz -e "ssh -p $DEPLOY_PORT" \ - scripts/deploy.sh scripts/deploy-lib.sh scripts/reconcile-prod-infra.sh \ + scripts/deploy.sh scripts/deploy-lib.sh scripts/reconcile-prod-infra.sh scripts/db-migrate-if-needed.sh \ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/scripts/" if [[ -d flipt ]]; then rsync -avz -e "ssh -p $DEPLOY_PORT" \ diff --git a/README.md b/README.md index cc2b418e..6015c84e 100644 --- a/README.md +++ b/README.md @@ -28,14 +28,24 @@ ## 🏃🏾‍♂️ Run NyumatFlix Locally > [!IMPORTANT] -> Prerequisites: +> **Required** > -> - [Bun](https://bun.sh/) installed on your machine. -> - A [PostgreSQL](https://www.postgresql.org/) database. -> - A [TMDb](https://www.themoviedb.org/) API key. -> - A [Resend](https://resend.com/) API key. +> - [Bun](https://bun.sh/) `1.3+` (see `packageManager` in root `package.json`) +> - [Docker](https://www.docker.com/) — scrape stack on `bun run dev`, and first-time Movi player WASM build +> - [PostgreSQL](https://www.postgresql.org/) (or [Neon](https://neon.tech/)) +> - [TMDb](https://www.themoviedb.org/) API key +> - [Resend](https://resend.com/) API key (`AUTH_RESEND_KEY`) +> - `AUTH_SECRET` — e.g. `openssl rand -base64 32` +> +> **Optional (feature-specific)** +> +> - `ID_MOE_API_KEY` — anime ↔ TMDB mapping +> - `MAL_CLIENT_ID` / `MAL_CLIENT_SECRET` — MyAnimeList sync on login +> - `scripts/gluetun/.env` — Surfshark (or other) WireGuard creds for scrape VPN egress +> - `CALLUSPIRATES_*` — Direct playback provider (nyumatflix.com only in prod) +> - `bun run cap:up` — self-hosted Cap instead of dev bypass on `/login` -To run the project on your machine, follow the steps below: +Monorepo layout: app in `apps/web`, shared packages in `packages/*`. Env lives at **repo root** (`.env.local`); `apps/web` symlinks to it on build/dev. 1. Clone or fork the repository @@ -44,59 +54,65 @@ git clone git@github.com:Nyumat/NyumatFlix.git cd NyumatFlix ``` -2. Next, create a `.env.local` file in the root directory of the project and add the following environment variables: +2. Environment ```bash -TMDB_API_KEY= -ID_MOE_API_KEY= -AUTH_SECRET= -AUTH_RESEND_KEY= -APP_URL= -AUTH_URL= -NEXTAUTH_URL= -RESEND_FROM_EMAIL= -DATABASE_URL= +cp .env.example .env.local ``` -You can get your own instance of the API keys by creating an account on [TMDb](https://www.themoviedb.org/) and [Resend](https://resend.com/). +Fill in at minimum: `TMDB_API_KEY`, `DATABASE_URL`, `AUTH_SECRET`, `AUTH_RESEND_KEY`. Local URLs are prefilled (`APP_URL` / `AUTH_URL` / `NEXTAUTH_URL` → `http://localhost:3000`). See `.env.example` for scrape (`FLARESOLVERR_URL`, `SCRAPE_PROXY_URL`), feature flags (`FLIPT_*`), and Cap. > [!TIP] -> For a database, you can use any PostgreSQL-compatible database you want. I recommend using [Neon](https://neon.tech/) for production and [Local Postgres](https://www.postgresql.org/) for development. - -For local development, set `APP_URL`, `AUTH_URL`, and `NEXTAUTH_URL` to `http://localhost:3000`. +> Any Postgres-compatible host works. Neon is fine for dev and prod. 3. Install dependencies -Bun is the package manager of choice here, so you'll need to install it first. You can do so by following the instructions [here](https://bun.sh/docs/installation). - ```bash bun install ``` -4. Configure magic strings +4. Configure magic strings (optional) -There's a few places in the codebase where I'm hardcoding some strings that you might want to change. You can find some of them in `lib/constants.ts`. Additionally, you might want to modify the email templates in `emails/` and `metadata` objects in `app/` and `layout/`. +Hardcoded branding/metadata: `apps/web/lib/constants.ts`, `apps/web/emails/`, and `metadata` in `apps/web/app/`. > [!NOTE] -> If you end up not changing the magic strings, that's fine too, but I'd appreciate credit if you re-deploy the project without modifications! +> If you ship without changing these, credit is appreciated. -5. Set up the database +5. Database ```bash -bun run db:generate -bun run db:push -bun run db:studio # tbh this is optional +cd apps/web && bun run db:migrate +# optional: bun run db:studio ``` -These will generate the necessary migration files and push the schema to your local (or remote) PostgreSQL database instance. +Fresh schema only (no migration history): `bun run db:push`. After pulling schema changes: `bun run db:migrate`. Existing DBs from older deploys: `bun run db:ensure-baseline` then `db:migrate`. + +6. Local scrape stack (optional but recommended for playback) -6. Run the development server +`bun run dev` runs `scripts/bootstrap-scrape-vpn.sh ensure-local` first — starts **FlareSolverr** (`:8191`), **Flipt** (`:8090`), and **Gluetun** (`:8888` proxy, `:8000` control) when Docker is available. Writes scrape-related vars into `.env.local`. + +```bash +# full bootstrap + prod env sync (needs SSH host `leetbot` unless you skip) +bun run dev:stack + +# scrape stack only, no Next.js +./scripts/bootstrap-scrape-vpn.sh ensure-local + +# skip scrape containers (metadata/browse still works; many scrapes will fail) +SKIP_SCRAPE_STACK=1 bun run dev +``` + +Gluetun needs VPN creds in `scripts/gluetun/.env` (copy from `scripts/gluetun/.env.example`). Without VPN, FlareSolverr still helps; vixsrc/vidsrc/vidrock often need the proxy. + +7. Run the development server ```bash bun run dev ``` -7. Finally, open [http://localhost:3000](http://localhost:3000) in your browser to see the project in action! +First run may build `@nyumatflix/player` via Docker if `public/vendor/player/` is missing. + +8. Open [http://localhost:3000](http://localhost:3000) +
+ + + diff --git a/packages/deploy-desk/package.json b/packages/deploy-desk/package.json new file mode 100644 index 00000000..4c86ed9a --- /dev/null +++ b/packages/deploy-desk/package.json @@ -0,0 +1,32 @@ +{ + "name": "@nyumatflix/deploy-desk", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "bun src/dev.ts", + "server:dev": "bun --watch src/server.ts", + "build": "vite build && bun build src/server.ts --target=bun --outdir=dist-server", + "start": "bun dist-server/server.js", + "typecheck": "tsc --project tsconfig.json --noEmit", + "lint": "biome check src vite.config.ts", + "test": "bun test" + }, + "dependencies": { + "@vitejs/plugin-react": "^4.3.1", + "hono": "^4.12.0", + "lucide-react": "^0.544.0", + "react": "^19.2.0", + "react-dom": "^19.2.0", + "tailwindcss": "^4.2.4", + "@tailwindcss/vite": "^4.2.4", + "vite": "^5.4.0" + }, + "devDependencies": { + "@types/bun": "^1.3.5", + "@biomejs/biome": "2.2.4", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "typescript": "^5.9.3" + } +} diff --git a/packages/deploy-desk/src/api.ts b/packages/deploy-desk/src/api.ts new file mode 100644 index 00000000..67ab985d --- /dev/null +++ b/packages/deploy-desk/src/api.ts @@ -0,0 +1,251 @@ +import { Hono } from "hono"; +import { bodyLimit } from "hono/body-limit"; +import { streamSSE } from "hono/streaming"; +import { apiPort, deskPort, productionUrl, sshHost } from "./lib/config"; +import { startDeploy, startRollback } from "./lib/deploy"; +import { getGitMeta } from "./lib/git"; +import { getJob, getJobs } from "./lib/jobs"; +import { + buildPreview, + refreshPreview, + startPreview, + stopPreview, +} from "./lib/preview"; +import { fetchHistory, fetchLive, fetchServices } from "./lib/remote"; + +const defaults = { + getGitMeta, + getJob, + getJobs, + buildPreview, + refreshPreview, + startPreview, + stopPreview, + fetchHistory, + fetchLive, + fetchServices, + startDeploy, + startRollback, +}; +export function createApp(overrides: Partial = {}) { + const deps = { ...defaults, ...overrides }; + const token = crypto.randomUUID(); + const app = new Hono(); + const hosts = new Set(["127.0.0.1:" + deskPort, "127.0.0.1:" + apiPort]); + const origins = new Set([...hosts].map((host) => "http://" + host)); + app.use("*", async (c, next) => { + const url = new URL(c.req.url); + if (!hosts.has(c.req.header("Host") ?? url.host)) + return c.json({ error: "Invalid local host" }, 403); + const origin = c.req.header("Origin"); + if (origin && !origins.has(origin)) + return c.json({ error: "Cross-origin access is not allowed" }, 403); + c.header("Cache-Control", "no-store"); + c.header("X-Content-Type-Options", "nosniff"); + c.header("X-Frame-Options", "DENY"); + c.header("Referrer-Policy", "no-referrer"); + c.header("Content-Security-Policy", "frame-ancestors 'none'"); + if (!["GET", "HEAD"].includes(c.req.method)) { + if (!origin || c.req.header("X-Deploy-Desk-Token") !== token) + return c.json({ error: "Refresh the desk before making changes" }, 403); + if (c.req.header("Content-Type")?.split(";")[0] !== "application/json") + return c.json({ error: "JSON body required" }, 415); + } + await next(); + }); + app.use( + "/api/*", + bodyLimit({ + maxSize: 4096, + onError: (c) => c.json({ error: "Request body is too large" }, 413), + }), + ); + app.onError((error, c) => c.json({ error: error.message }, 500)); + app.get("/api/session", (c) => + c.json({ token, remote: sshHost, productionUrl }), + ); + app.get("/api/health", (c) => c.json({ ok: true })); + app.get("/api/git", (c) => c.json(deps.getGitMeta())); + app.get("/api/status", async (c) => { + const [live, services] = await Promise.allSettled([ + deps.fetchLive(), + deps.fetchServices(), + ]); + const errors = [live, services].flatMap((result) => + result.status === "rejected" + ? [ + result.reason instanceof Error + ? result.reason.message + : "Remote status unavailable", + ] + : [], + ); + return c.json({ + live: live.status === "fulfilled" ? live.value : null, + services: services.status === "fulfilled" ? services.value : [], + remote: sshHost, + productionUrl, + errors, + checkedAt: new Date().toISOString(), + }); + }); + app.get("/api/history", async (c) => { + const limit = Number(c.req.query("limit") ?? 20); + const offset = Number(c.req.query("offset") ?? 0); + if ( + !Number.isInteger(limit) || + limit < 1 || + limit > 100 || + !Number.isInteger(offset) || + offset < 0 || + offset > 1000 + ) + return c.json({ error: "Invalid pagination" }, 400); + try { + const entries = await deps.fetchHistory(limit, offset); + return c.json({ + entries, + nextOffset: + entries.length === limit && offset + limit <= 1000 + ? offset + limit + : null, + }); + } catch (error) { + return c.json( + { + error: error instanceof Error ? error.message : "History unavailable", + }, + 502, + ); + } + }); + app.get("/api/preview", async (c) => c.json(await deps.refreshPreview())); + app.get("/api/jobs", (c) => + c.json({ jobs: deps.getJobs().map(({ logs: _logs, ...job }) => job) }), + ); + app.get("/api/jobs/:id", (c) => { + const job = deps.getJob(c.req.param("id")); + return job ? c.json(job) : c.json({ error: "Job not found" }, 404); + }); + for (const action of ["build", "start", "stop"] as const) { + app.post("/api/preview/" + action, async (c) => { + try { + const task = { + build: deps.buildPreview, + start: deps.startPreview, + stop: deps.stopPreview, + }[action]; + return c.json({ job: await task() }, 202); + } catch (error) { + return c.json( + { + error: + error instanceof Error ? error.message : "Preview action failed", + }, + 409, + ); + } + }); + } + app.post("/api/deploy", async (c) => { + const body: unknown = await c.req.json().catch(() => null); + if ( + !body || + typeof body !== "object" || + !("confirmed" in body) || + body.confirmed !== true || + !("fingerprint" in body) || + typeof body.fingerprint !== "string" || + !("fast" in body) || + typeof body.fast !== "boolean" || + !("skipPreview" in body) || + typeof body.skipPreview !== "boolean" + ) + return c.json({ error: "Review and confirm the deployment first" }, 400); + try { + return c.json( + { + job: await deps.startDeploy({ + fast: body.fast, + skipPreview: body.skipPreview, + fingerprint: body.fingerprint, + }), + }, + 202, + ); + } catch (error) { + return c.json( + { + error: + error instanceof Error ? error.message : "Deploy could not start", + }, + 409, + ); + } + }); + app.post("/api/rollback", async (c) => { + const body: unknown = await c.req.json().catch(() => null); + if ( + !body || + typeof body !== "object" || + !("confirmed" in body) || + body.confirmed !== true || + !("sha" in body) || + typeof body.sha !== "string" || + !/^[a-f0-9]{7,40}(?:-dirty)?$/i.test(body.sha) || + !("image" in body) || + typeof body.image !== "string" || + !("deployedAt" in body) || + typeof body.deployedAt !== "string" + ) + return c.json( + { error: "An exact history entry and confirmation are required" }, + 400, + ); + try { + return c.json( + { + job: await deps.startRollback(body.sha, body.image, body.deployedAt), + }, + 202, + ); + } catch (error) { + return c.json( + { + error: + error instanceof Error ? error.message : "Rollback could not start", + }, + 409, + ); + } + }); + app.get("/api/jobs/:id/events", (c) => { + const id = c.req.param("id"); + if (!deps.getJob(id)) return c.json({ error: "Job not found" }, 404); + const lastId = Number(c.req.header("Last-Event-ID") ?? -1); + return streamSSE(c, async (stream) => { + let cursor = Number.isSafeInteger(lastId) ? lastId + 1 : 0; + while (!stream.aborted) { + const job = deps.getJob(id); + if (!job) return; + cursor = Math.max(cursor, job.logOffset); + for (; cursor < job.logOffset + job.logs.length; cursor++) { + await stream.writeSSE({ + event: "log", + id: String(cursor), + data: JSON.stringify(job.logs[cursor - job.logOffset]), + }); + } + const { logs: _logs, ...status } = job; + await stream.writeSSE({ + event: "status", + data: JSON.stringify(status), + }); + if (job.status === "succeeded" || job.status === "failed") return; + await stream.sleep(500); + } + }); + }); + app.all("/api/*", (c) => c.json({ error: "API route not found" }, 404)); + return app; +} diff --git a/packages/deploy-desk/src/dev.ts b/packages/deploy-desk/src/dev.ts new file mode 100644 index 00000000..e7c922f7 --- /dev/null +++ b/packages/deploy-desk/src/dev.ts @@ -0,0 +1,87 @@ +import { connect } from "node:net"; +import { createServer } from "vite"; +import { apiPort, deskPort, packageRoot } from "./lib/config"; + +async function portInUse(port: number) { + return new Promise((resolve) => { + const socket = connect({ host: "127.0.0.1", port }); + socket.setTimeout(1000); + socket.once("connect", () => { + socket.destroy(); + resolve(true); + }); + socket.once("error", () => { + socket.destroy(); + resolve(false); + }); + socket.once("timeout", () => { + socket.destroy(); + resolve(true); + }); + }); +} +for (const port of [deskPort, apiPort]) { + if (await portInUse(port)) + throw new Error( + "Port " + port + " is already in use; the desk was not started", + ); +} +const api = Bun.spawn([process.execPath, "src/server.ts"], { + cwd: packageRoot, + env: { ...process.env, DEPLOY_DESK_DEV: "1" }, + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", +}); +let vite: Awaited> | undefined; +let stopping = false; +async function stop(code = 0) { + if (stopping) return; + stopping = true; + api.kill("SIGTERM"); + await vite?.close(); + await api.exited; + process.exit(code); +} +process.on("SIGINT", () => { + void stop(); +}); +process.on("SIGTERM", () => { + void stop(); +}); +void api.exited.then((code) => { + if (!stopping) void stop(code || 1); +}); +try { + // API startup must succeed before opening the browser. + let ready = false; + for (let attempt = 0; attempt < 50; attempt++) { + try { + ready = (await fetch("http://127.0.0.1:" + apiPort + "/api/health")).ok; + } catch { + /* Starting. */ + } + if (ready) break; + await Bun.sleep(100); + } + if (!ready) throw new Error("Deploy Desk API did not start"); + vite = await createServer({ + root: packageRoot, + configFile: packageRoot + "/vite.config.ts", + }); + await vite.listen(); + const url = "http://127.0.0.1:" + deskPort; + console.log("Deploy Desk → " + url); + if (!process.argv.includes("--no-open")) { + Bun.spawn([process.platform === "darwin" ? "open" : "xdg-open", url], { + stdout: "ignore", + stderr: "ignore", + }); + } + await new Promise(() => { + // Keep the Vite process alive until a signal or API exit requests shutdown. + }); +} catch (error) { + console.error(error); + await stop(1); +} diff --git a/packages/deploy-desk/src/lib/config.ts b/packages/deploy-desk/src/lib/config.ts new file mode 100644 index 00000000..4602dd1b --- /dev/null +++ b/packages/deploy-desk/src/lib/config.ts @@ -0,0 +1,72 @@ +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { existsSync, readFileSync } from "node:fs"; + +const requireSafeToken = (value: string, label: string, pattern: RegExp) => { + if (!pattern.test(value)) { + throw new Error(`Invalid ${label}: ${value}`); + } + return value; +}; + +const numberInRange = (raw: string | undefined, fallback: number) => { + const value = Number(raw ?? fallback); + if (!Number.isInteger(value) || value < 1 || value > 65_535) { + throw new Error(`Invalid port: ${raw}`); + } + return value; +}; + +function findRoot() { + let directory = dirname(fileURLToPath(import.meta.url)); + while (true) { + const manifest = resolve(directory, "package.json"); + if ( + existsSync(manifest) && + JSON.parse(readFileSync(manifest, "utf8")).name === "nyumatflix" + ) + return directory; + const parent = resolve(directory, ".."); + if (parent === directory) + throw new Error("Run Deploy Desk inside the NyumatFlix checkout"); + directory = parent; + } +} +export const repoRoot = findRoot(); +export const packageRoot = resolve(repoRoot, "packages/deploy-desk"); +export const deskPort = numberInRange(process.env.DEPLOY_DESK_PORT, 4545); +export const apiPort = numberInRange( + process.env.DEPLOY_DESK_API_PORT, + deskPort + 1, +); +export const host = process.env.DEPLOY_DESK_HOST ?? "127.0.0.1"; +export const previewPort = numberInRange(process.env.PREVIEW_PORT, 9080); +export const sshHost = requireSafeToken( + process.env.SSH_HOST ?? "leetbot", + "SSH_HOST", + /^[a-zA-Z0-9][a-zA-Z0-9_.@-]*$/, +); +export const remoteAppDir = requireSafeToken( + process.env.REMOTE_APP_DIR ?? "apps/nyumatflix", + "REMOTE_APP_DIR", + /^[a-zA-Z0-9_][a-zA-Z0-9_./-]*$/, +); +export const productionUrl = + process.env.DEPLOY_DESK_PROD_URL ?? "https://nyumatflix.com"; +export const previewImage = "nyumatflix-preview:local"; +export const previewContainer = "nyumatflix-preview-local"; +export const statePath = resolve(repoRoot, ".deploy-desk/state.json"); + +if (host !== "127.0.0.1") { + throw new Error("DEPLOY_DESK_HOST must be 127.0.0.1; the desk is local-only"); +} +if ( + remoteAppDir.split("/").some((part) => !part || part === "." || part === "..") +) + throw new Error( + "REMOTE_APP_DIR must be a relative path below the remote home", + ); +if (deskPort === apiPort || [deskPort, apiPort].includes(previewPort)) + throw new Error("Desk, API, and preview ports must differ"); +if (!/^https?:$/.test(new URL(productionUrl).protocol)) + throw new Error("Invalid production URL"); diff --git a/packages/deploy-desk/src/lib/deploy.ts b/packages/deploy-desk/src/lib/deploy.ts new file mode 100644 index 00000000..a2e9c872 --- /dev/null +++ b/packages/deploy-desk/src/lib/deploy.ts @@ -0,0 +1,117 @@ +import { repoRoot, sshHost, remoteAppDir } from "./config"; +import { getGitMeta, resolveDeployIdentity } from "./git"; +import { getStoredPreview, startJob } from "./jobs"; +import { inspectPreview } from "./preview"; +import { runProcess } from "./process"; +import { fetchHistory, remoteServe } from "./remote"; + +export const startDeploy = (options: { + fast: boolean; + skipPreview: boolean; + fingerprint: string; +}) => + startJob("deploy", async ({ log, phase }) => { + const meta = getGitMeta(); + if (meta.fingerprint !== options.fingerprint) + throw new Error( + "Checkout changed since review. Refresh and confirm again.", + ); + if (!options.skipPreview) { + phase("Verify preview"); + const { image, container } = await inspectPreview(); + const [imageId, fingerprint] = image.stdout.trim().split("|"); + const [, status, health, label, runningImage] = container.stdout + .trim() + .split("|"); + if ( + !image.ok || + !container.ok || + label !== "local" || + status !== "running" || + health !== "healthy" || + imageId !== runningImage || + fingerprint !== meta.fingerprint || + getStoredPreview().fingerprint !== meta.fingerprint + ) { + throw new Error( + "A healthy preview of this checkout is required. Build and start it, or explicitly skip the preview.", + ); + } + } + const deploy = resolveDeployIdentity(meta); + const image = "whotypes/nyumatflix:" + deploy.sha; + const env: Record = { + DEPLOY_SHA: deploy.sha, + DEPLOY_SHORT_SHA: deploy.shortSha, + DEPLOY_MESSAGE: deploy.message, + DEPLOY_AUTHOR: deploy.author, + DEPLOY_SOURCE: deploy.source, + DOCKER_IMAGE: image, + SSH_HOST: sshHost, + REMOTE_APP_DIR: remoteAppDir, + }; + if (options.fast) { + env.SKIP_PLAYER_BUILD = "1"; + env.SKIP_SCRAPE_STACK = "1"; + } + phase( + options.fast ? "Build and push image (fast)" : "Build and push image", + ); + const build = await runProcess(repoRoot + "/scripts/deploy.sh", ["bp"], { + env, + onLine: log, + }); + if (!build.ok) throw new Error("Image build/push exited " + build.exitCode); + if (getGitMeta().fingerprint !== meta.fingerprint) + throw new Error( + "Checkout changed during build. Image was pushed, but production rollout was stopped. Review and retry.", + ); + phase("Sync production environment"); + const sync = await runProcess( + repoRoot + "/scripts/sync-prod-env.sh", + ["push"], + { env, onLine: log }, + ); + if (!sync.ok) throw new Error("Environment sync exited " + sync.exitCode); + phase("Roll production container"); + const remote = await remoteServe({ ...deploy, image }, "local", log); + if (!remote.ok) + throw new Error( + "Remote rollout exited " + + remote.exitCode + + ". Check production status before retrying.", + ); + }); + +export const startRollback = (sha: string, image: string, deployedAt: string) => + startJob("rollback", async ({ log, phase }) => { + phase("Verify rollback target"); + // Resolve all metadata server-side; never execute a client-supplied image or command. + let target; + for (let offset = 0; offset <= 1000; offset += 100) { + const history = await fetchHistory(100, offset); + target = history.find( + (entry) => + entry.sha === sha && + entry.image === image && + entry.deployedAt === deployedAt, + ); + if (target || history.length < 100) break; + } + if (!target) + throw new Error("Rollback target is no longer present in remote history"); + if ( + !/^whotypes\/nyumatflix(?::[a-zA-Z0-9_.-]+|@sha256:[a-f0-9]{64})$/.test( + target.image, + ) + ) + throw new Error("Rollback image is outside the NyumatFlix repository"); + phase("Roll back to " + target.shortSha); + const remote = await remoteServe(target, "rollback", log); + if (!remote.ok) + throw new Error( + "Remote rollback exited " + + remote.exitCode + + ". Check production status before retrying.", + ); + }); diff --git a/packages/deploy-desk/src/lib/git.ts b/packages/deploy-desk/src/lib/git.ts new file mode 100644 index 00000000..9bf0535b --- /dev/null +++ b/packages/deploy-desk/src/lib/git.ts @@ -0,0 +1,47 @@ +import type { GitMeta } from "./types"; +import { runSync } from "./process"; +import { createHash } from "node:crypto"; + +const git = (...args: string[]) => { + const result = runSync("git", args); + if (!result.ok) + throw new Error(result.stderr.trim() || `git ${args[0]} failed`); + return result.stdout.trim(); +}; + +export const getGitMeta = (): GitMeta => { + const status = git("status", "--porcelain"); + const unstaged = git("diff", "--stat"); + const staged = git("diff", "--cached", "--stat"); + const diffStat = + [unstaged, staged].filter(Boolean).join("\n") || "No tracked diff"; + const fingerprint = createHash("sha256") + .update(git("rev-parse", "HEAD")) + .update(git("diff", "HEAD", "--binary")) + .update(status); + const untracked = git("ls-files", "--others", "--exclude-standard", "-z") + .split("\0") + .filter(Boolean); + for (const file of untracked) + fingerprint.update(file).update(git("hash-object", "--", file)); + return { + branch: git("branch", "--show-current") || "detached HEAD", + sha: git("rev-parse", "HEAD"), + shortSha: git("rev-parse", "--short=7", "HEAD"), + message: git("log", "-1", "--pretty=format:%s"), + author: git("log", "-1", "--pretty=format:%an"), + dirty: status.length > 0, + diffStat, + changedFiles: status ? status.split("\n").length : 0, + fingerprint: fingerprint.digest("hex"), + }; +}; + +export const resolveDeployIdentity = (meta: GitMeta) => ({ + sha: meta.dirty ? `${meta.sha}-dirty` : meta.sha, + shortSha: meta.dirty ? `${meta.shortSha}+` : meta.shortSha, + message: meta.dirty ? `${meta.message} (uncommitted)` : meta.message, + author: meta.author, + source: "local", + dirty: meta.dirty, +}); diff --git a/packages/deploy-desk/src/lib/jobs.ts b/packages/deploy-desk/src/lib/jobs.ts new file mode 100644 index 00000000..51996cd8 --- /dev/null +++ b/packages/deploy-desk/src/lib/jobs.ts @@ -0,0 +1,200 @@ +import { dirname } from "node:path"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { previewImage, previewPort, statePath } from "./config"; +import type { Job, JobKind, PreviewState } from "./types"; + +type StoredState = { jobs: Job[]; preview: PreviewState }; +type JobContext = { + log: (line: string) => void; + phase: (name: string) => void; +}; +export class JobStore { + private state: StoredState = { + jobs: [], + preview: { + status: "idle", + port: previewPort, + url: "http://127.0.0.1:" + previewPort, + image: previewImage, + }, + }; + private writes: Promise = Promise.resolve(); + private active: string | null = null; + private timer?: ReturnType; + constructor( + private path: string, + private redact: (value: string) => string = (value) => value, + ) {} + + async persist() { + const snapshot = JSON.stringify(this.state); + this.writes = this.writes + .catch(() => { + // A failed prior write must not prevent the next state snapshot. + }) + .then(async () => { + await mkdir(dirname(this.path), { recursive: true, mode: 0o700 }); + await writeFile(this.path + ".tmp", snapshot, { mode: 0o600 }); + await rename(this.path + ".tmp", this.path); + }); + return this.writes; + } + async load() { + try { + const saved: StoredState = JSON.parse(await readFile(this.path, "utf8")); + if (!Array.isArray(saved.jobs) || !saved.preview) + throw new Error("Invalid state file"); + this.state.jobs = saved.jobs + .filter((job) => typeof job.id === "string" && Array.isArray(job.logs)) + .slice(0, 30); + this.state.preview = { + ...saved.preview, + port: previewPort, + url: "http://127.0.0.1:" + previewPort, + image: previewImage, + }; + for (const job of this.state.jobs) { + job.logOffset ??= 0; + if (job.status === "running" || job.status === "queued") { + job.status = "failed"; + job.error = + "Desk interrupted. Check production status before retrying; a remote rollout may have continued."; + job.finishedAt = new Date().toISOString(); + } + } + if (["building", "starting"].includes(this.state.preview.status)) + this.state.preview.status = "failed"; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await this.persist(); + } + get(id: string) { + return this.state.jobs.find((job) => job.id === id); + } + list() { + return this.state.jobs; + } + busy() { + return this.active !== null; + } + preview() { + return this.state.preview; + } + async setPreview(patch: Partial) { + this.state.preview = { + ...this.state.preview, + ...patch, + updatedAt: new Date().toISOString(), + }; + await this.persist(); + } + async start(kind: JobKind, task: (context: JobContext) => Promise) { + // Both preview and deploy use this checkout and its Docker artifacts. + if (this.active) + throw new Error("Another job is running. Wait for it to finish."); + const job: Job = { + id: crypto.randomUUID(), + kind, + status: "queued", + phase: "Queued", + startedAt: new Date().toISOString(), + logs: [], + logOffset: 0, + }; + this.active = job.id; + this.state.jobs = [job, ...this.state.jobs].slice(0, 30); + try { + await this.persist(); + } catch (error) { + this.active = null; + this.state.jobs.shift(); + throw error; + } + const log = (text: string) => { + const line = this.redact(text) + .replace(new RegExp("\\u001b\\[[0-9;]*[a-zA-Z]", "g"), "") + .slice(0, 8_192); + if (!line) return; + job.logs.push(line); + if (job.logs.length > 1_000) { + job.logs.shift(); + job.logOffset++; + } + if (!this.timer) + this.timer = setTimeout(() => { + this.timer = undefined; + void this.persist().catch((error) => + console.error("Could not persist logs:", error.message), + ); + }, 500); + }; + void (async () => { + job.status = "running"; + try { + await task({ + log, + phase: (name) => { + job.phase = name; + log("→ " + name); + }, + }); + job.status = "succeeded"; + job.phase = "Complete"; + } catch (error) { + job.status = "failed"; + job.phase = "Failed"; + job.error = this.redact( + error instanceof Error ? error.message : "Job failed", + ); + log(job.error); + } finally { + job.finishedAt = new Date().toISOString(); + job.durationMs = Date.now() - Date.parse(job.startedAt); + clearTimeout(this.timer); + this.timer = undefined; + try { + await this.persist(); + } catch (error) { + console.error("Could not persist job:", error); + } + this.active = null; + } + })(); + return job; + } +} + +const secrets = new Set(); +export async function loadRedactions(paths: string[]) { + for (const path of paths) { + try { + for (const line of (await readFile(path, "utf8")).split("\n")) { + const match = line.match(/^\s*(?:export\s+)?([\w]+)\s*=\s*(.*?)\s*$/); + if ( + match?.[1] && + /KEY|TOKEN|SECRET|PASSWORD|DATABASE|CREDENTIAL/i.test(match[1]) + ) { + const value = (match[2] ?? "").replace(/^["']|["']$/g, ""); + if (value.length >= 6) secrets.add(value); + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } +} +function redact(text: string) { + for (const secret of secrets) text = text.split(secret).join("[redacted]"); + return text.replace(/(https?:\/\/)[^\s/@]+:[^\s/@]+@/g, "$1[redacted]@"); +} +export const jobs = new JobStore(statePath, redact); +export const getJob = (id: string) => jobs.get(id); +export const getJobs = () => jobs.list(); +export const getStoredPreview = () => jobs.preview(); +export const setStoredPreview = (patch: Partial) => + jobs.setPreview(patch); +export const startJob = ( + kind: JobKind, + task: (context: JobContext) => Promise, +) => jobs.start(kind, task); diff --git a/packages/deploy-desk/src/lib/preview.ts b/packages/deploy-desk/src/lib/preview.ts new file mode 100644 index 00000000..132b1fc9 --- /dev/null +++ b/packages/deploy-desk/src/lib/preview.ts @@ -0,0 +1,189 @@ +import { + previewContainer, + previewImage, + previewPort, + repoRoot, +} from "./config"; +import { getGitMeta } from "./git"; +import { jobs, getStoredPreview, setStoredPreview, startJob } from "./jobs"; +import { runProcess } from "./process"; + +export const previewEnvironment = () => ({ + PREVIEW_PORT: String(previewPort), + PREVIEW_ENV_FILE: + process.env.DEPLOY_DESK_PREVIEW_ENV ?? repoRoot + "/.env.prod", +}); +export const inspectPreview = async () => { + const [image, container] = await Promise.all([ + runProcess( + "docker", + [ + "image", + "inspect", + "--format", + '{{.Id}}|{{index .Config.Labels "nyumatflix.preview.fingerprint"}}', + previewImage, + ], + { timeoutMs: 8_000 }, + ), + runProcess( + "docker", + [ + "container", + "inspect", + "--format", + '{{.Id}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{index .Config.Labels "nyumatflix.preview"}}|{{.Image}}|{{json .NetworkSettings.Ports}}', + previewContainer, + ], + { timeoutMs: 8_000 }, + ), + ]); + return { image, container }; +}; +export async function refreshPreview() { + if (jobs.busy()) return getStoredPreview(); + try { + const { image, container } = await inspectPreview(); + const [imageId, fingerprint] = image.ok + ? image.stdout.trim().split("|") + : []; + const [containerId, status, health, label, runningImageId, bindings] = + container.stdout.trim().split("|"); + let boundPort: string | undefined; + if (container.ok && bindings) { + const ports = JSON.parse(bindings) as Record< + string, + { HostIp: string; HostPort: string }[] | null + >; + boundPort = ports["8080/tcp"]?.find( + (entry) => entry.HostIp === "127.0.0.1", + )?.HostPort; + } + if (container.ok && label !== "local") + throw new Error( + "Container name belongs to an unmanaged container; it will not be touched", + ); + if (jobs.busy()) return getStoredPreview(); // Do not overwrite a job that started during inspection. + const running = container.ok && status === "running"; + await setStoredPreview({ + status: running ? "running" : image.ok ? "ready" : "idle", + imageId, + fingerprint: + imageId && running && runningImageId !== imageId + ? undefined + : fingerprint, + containerId: container.ok ? containerId : undefined, + health: running ? health : undefined, + port: boundPort ? Number(boundPort) : previewPort, + url: "http://127.0.0.1:" + (boundPort ?? previewPort), + error: + !image.ok && !/No such (image|object)/i.test(image.stderr) + ? "Docker is unavailable. Start Docker and refresh." + : undefined, + }); + } catch (error) { + if (!jobs.busy()) + await setStoredPreview({ + status: "failed", + health: undefined, + error: error instanceof Error ? error.message : "Docker is unavailable", + }); + } + return getStoredPreview(); +} + +export const buildPreview = () => + startJob("preview-build", async ({ log, phase }) => { + await setStoredPreview({ + status: "building", + fingerprint: undefined, + error: undefined, + }); + try { + const fingerprint = getGitMeta().fingerprint; + phase("Build local production image"); + const result = await runProcess( + repoRoot + "/scripts/deploy.sh", + ["preview-build"], + { + env: { + ...previewEnvironment(), + DEPLOY_DESK_FINGERPRINT: fingerprint, + }, + onLine: log, + }, + ); + if (!result.ok) + throw new Error("Preview build exited " + result.exitCode); + if (getGitMeta().fingerprint !== fingerprint) + throw new Error( + "Checkout changed during the build. Review the changes and build again before using the preview gate.", + ); + const { image } = await inspectPreview(); + if (!image.ok) throw new Error("Built image could not be inspected"); + await setStoredPreview({ + status: "ready", + imageId: image.stdout.trim().split("|")[0], + fingerprint, + }); + } catch (error) { + await setStoredPreview({ status: "failed", fingerprint: undefined }); + throw error; + } + }); + +export const startPreview = () => + startJob("preview-start", async ({ log, phase }) => { + await setStoredPreview({ status: "starting", error: undefined }); + try { + phase("Start isolated preview container"); + const result = await runProcess( + repoRoot + "/scripts/deploy.sh", + ["preview-serve"], + { env: previewEnvironment(), onLine: log }, + ); + if (!result.ok) + throw new Error("Preview start exited " + result.exitCode); + const { image, container } = await inspectPreview(); + const [imageId, fingerprint] = image.stdout.trim().split("|"); + const [containerId, status, health, label, runningImageId] = + container.stdout.trim().split("|"); + if ( + !container.ok || + status !== "running" || + health !== "healthy" || + label !== "local" || + runningImageId !== imageId + ) + throw new Error("Preview health or image verification failed"); + await setStoredPreview({ + status: "running", + imageId, + fingerprint, + containerId, + health, + port: previewPort, + url: "http://127.0.0.1:" + previewPort, + }); + } catch (error) { + await setStoredPreview({ status: "failed", health: undefined }); + throw error; + } + }); + +export const stopPreview = () => + startJob("preview-stop", async ({ log, phase }) => { + phase("Stop isolated preview container"); + const result = await runProcess( + repoRoot + "/scripts/deploy.sh", + ["preview-stop"], + { onLine: log }, + ); + if (!result.ok) throw new Error("Preview stop exited " + result.exitCode); + await setStoredPreview({ + status: getStoredPreview().imageId ? "ready" : "idle", + containerId: undefined, + health: undefined, + error: undefined, + }); + }); diff --git a/packages/deploy-desk/src/lib/process.ts b/packages/deploy-desk/src/lib/process.ts new file mode 100644 index 00000000..683fbf0c --- /dev/null +++ b/packages/deploy-desk/src/lib/process.ts @@ -0,0 +1,111 @@ +import { spawn } from "node:child_process"; +import { repoRoot } from "./config"; + +export type ProcessResult = { + ok: boolean; + exitCode: number; + stdout: string; + stderr: string; +}; +type RunOptions = { + cwd?: string; + env?: Record; + onLine?: (line: string) => void; + timeoutMs?: number; +}; +const children = new Set(); +export function stopProcesses() { + for (const pid of children) { + try { + process.kill(-pid, "SIGTERM"); + } catch { + /* Already exited. */ + } + } +} + +export function runProcess( + command: string, + args: string[], + options: RunOptions = {}, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd ?? repoRoot, + env: { ...process.env, ...options.env }, + stdio: ["ignore", "pipe", "pipe"], + detached: true, + }); + if (child.pid) children.add(child.pid); + let stdout = ""; + let stderr = ""; + const attach = ( + stream: NodeJS.ReadableStream, + save: (text: string) => void, + ) => { + let pending = ""; + stream.setEncoding("utf8"); + stream.on("data", (text: string) => { + save(text); + pending += text; + const lines = pending.split(/[\r\n]+/); + pending = lines.pop() ?? ""; + for (const line of lines) options.onLine?.(line.slice(0, 8_192)); + if (pending.length > 8_192) { + options.onLine?.(pending.slice(0, 8_192)); + pending = ""; + } + }); + stream.on("end", () => { + if (pending) options.onLine?.(pending); + }); + }; + attach(child.stdout, (text) => { + stdout = (stdout + text).slice(-256_000); + }); + attach(child.stderr, (text) => { + stderr = (stderr + text).slice(-64_000); + }); + const timeout = setTimeout( + () => { + if (child.pid) { + try { + process.kill(-child.pid, "SIGKILL"); + } catch { + /* Already exited. */ + } + } + }, + options.timeoutMs ?? 45 * 60_000, + ); + const cleanup = () => { + clearTimeout(timeout); + if (child.pid) children.delete(child.pid); + }; + child.on("error", (error) => { + cleanup(); + reject(error); + }); + child.on("close", (code) => { + cleanup(); + resolve({ ok: code === 0, exitCode: code ?? -1, stdout, stderr }); + }); + }); +} + +export const runSync = (command: string, args: string[]): ProcessResult => { + const result = Bun.spawnSync([command, ...args], { + cwd: repoRoot, + env: process.env, + stdout: "pipe", + stderr: "pipe", + }); + return { + ok: result.exitCode === 0, + exitCode: result.exitCode, + stdout: result.stdout.toString(), + stderr: result.stderr.toString(), + }; +}; +export const shellQuote = (value: string) => + "'" + value.replace(/'/g, "'\\''") + "'"; diff --git a/packages/deploy-desk/src/lib/remote.ts b/packages/deploy-desk/src/lib/remote.ts new file mode 100644 index 00000000..ee28f5f9 --- /dev/null +++ b/packages/deploy-desk/src/lib/remote.ts @@ -0,0 +1,120 @@ +import { remoteAppDir, sshHost } from "./config"; +import { runProcess, shellQuote } from "./process"; +import type { DeployEntry, LiveDeploy, ServiceRow } from "./types"; + +const ssh = (command: string, onLine?: (line: string) => void) => + runProcess( + "ssh", + [ + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=8", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=3", + sshHost, + command, + ], + { onLine, timeoutMs: onLine ? 45 * 60_000 : 25_000 }, + ); + +const isDeployEntry = (value: unknown): value is DeployEntry => { + if (!value || typeof value !== "object") return false; + const row = value as Record; + return ( + typeof row.sha === "string" && + typeof row.shortSha === "string" && + typeof row.message === "string" && + typeof row.author === "string" && + typeof row.deployedAt === "string" && + typeof row.image === "string" && + typeof row.source === "string" && + typeof row.port === "number" + ); +}; + +export const parseHistory = (raw: string): DeployEntry[] => { + const entries: DeployEntry[] = []; + for (const line of raw.split("\n")) { + if (!line.trim()) continue; + try { + const parsed: unknown = JSON.parse(line); + if (isDeployEntry(parsed)) entries.push(parsed); + } catch { + // A partial history line should not take down the control plane. + } + } + return entries.reverse(); +}; + +export const fetchHistory = async (limit = 30, offset = 0) => { + if ( + !Number.isInteger(limit) || + !Number.isInteger(offset) || + limit < 1 || + offset < 0 + ) + throw new Error("Invalid history pagination"); + const safeLimit = Math.min(Math.max(limit, 1), 100); + const safeOffset = Math.min(Math.max(offset, 0), 1_000); + const result = await ssh( + `test -f ~/${remoteAppDir}/deployments.jsonl && tail -n ${safeLimit + safeOffset} ~/${remoteAppDir}/deployments.jsonl || true`, + ); + if (!result.ok) + throw new Error( + result.stderr.trim() || "Could not read deployment history", + ); + return parseHistory(result.stdout).slice(safeOffset, safeOffset + safeLimit); +}; + +export const fetchLive = async (): Promise => { + const result = await ssh( + `cd ~/${remoteAppDir} && ./scripts/deploy.sh current`, + ); + if (!result.ok) + throw new Error(result.stderr.trim() || "Could not read live deployment"); + const fields = new Map(); + for (const line of result.stdout.split("\n")) { + const index = line.indexOf("="); + if (index > 0) fields.set(line.slice(0, index), line.slice(index + 1)); + } + const shortSha = fields.get("shortSha") ?? ""; + if (!shortSha || shortSha === "") return null; + return { + sha: fields.get("sha") ?? "", + shortSha, + message: fields.get("message") ?? "", + author: fields.get("author") ?? "", + deployedAt: fields.get("deployedAt") ?? "", + source: fields.get("source") ?? "", + image: fields.get("image") ?? "", + }; +}; + +export const fetchServices = async (): Promise => { + const result = await ssh( + "sudo docker ps -a --filter name=^/nyumatflix$ --filter name=^/nyumatflix-imgproxy$ --filter name=^/cap$ --filter name=^/cap-valkey$ --filter name=^/flipt$ --filter name=^/gluetun$ --filter name=^/flaresolverr$ --format '{{.Names}}|{{.Status}}|{{.Image}}'", + ); + if (!result.ok) + throw new Error(result.stderr.trim() || "Could not read service status"); + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [name = "", status = "", image = ""] = line.split("|"); + return { name, status, image }; + }); +}; + +export const remoteServe = async ( + entry: Pick, + source: "local" | "rollback", + onLine: (line: string) => void, +) => + ssh( + `set -euo pipefail; cd ~/${remoteAppDir}; export NYUMATFLIX_ROOT=~/${remoteAppDir}; export DOCKER_IMAGE=${shellQuote(entry.image)}; export DEPLOY_SHA=${shellQuote(entry.sha)}; export DEPLOY_SHORT_SHA=${shellQuote(entry.shortSha)}; export DEPLOY_MESSAGE=${shellQuote(entry.message)}; export DEPLOY_AUTHOR=${shellQuote(entry.author)}; export DEPLOY_SOURCE=${shellQuote(source)}; ./scripts/deploy.sh serve`, + onLine, + ); diff --git a/packages/deploy-desk/src/lib/types.ts b/packages/deploy-desk/src/lib/types.ts new file mode 100644 index 00000000..03c98b67 --- /dev/null +++ b/packages/deploy-desk/src/lib/types.ts @@ -0,0 +1,66 @@ +export type DeployEntry = { + sha: string; + shortSha: string; + message: string; + author: string; + deployedAt: string; + image: string; + source: string; + port: number; + dirty?: boolean; +}; + +export type LiveDeploy = Omit; + +export type ServiceRow = { + name: string; + status: string; + image: string; +}; + +export type GitMeta = { + branch: string; + sha: string; + shortSha: string; + message: string; + author: string; + dirty: boolean; + diffStat: string; + changedFiles: number; + fingerprint: string; +}; + +export type JobKind = + | "deploy" + | "rollback" + | "preview-build" + | "preview-start" + | "preview-stop"; + +export type JobStatus = "queued" | "running" | "succeeded" | "failed"; + +export type Job = { + id: string; + kind: JobKind; + status: JobStatus; + phase: string; + startedAt: string; + finishedAt?: string; + durationMs?: number; + error?: string; + logs: string[]; + logOffset: number; +}; + +export type PreviewState = { + status: "idle" | "building" | "ready" | "starting" | "running" | "failed"; + port: number; + url: string; + image: string; + imageId?: string; + containerId?: string; + health?: string; + updatedAt?: string; + fingerprint?: string; + error?: string; +}; diff --git a/packages/deploy-desk/src/main.tsx b/packages/deploy-desk/src/main.tsx new file mode 100644 index 00000000..361a12cb --- /dev/null +++ b/packages/deploy-desk/src/main.tsx @@ -0,0 +1,1074 @@ +import { + useCallback, + useEffect, + useRef, + useState, + type ReactNode, +} from "react"; +import { createRoot } from "react-dom/client"; +import { + ArrowDownToLine, + ArrowUpRight, + Box, + Check, + ChevronRight, + Circle, + Clock, + ExternalLink, + GitBranch, + GitCommitHorizontal, + History, + LayoutDashboard, + LoaderCircle, + Play, + RefreshCw, + Rocket, + RotateCcw, + ShieldCheck, + Square, + Terminal, + X, +} from "lucide-react"; +import type { + DeployEntry, + GitMeta, + Job, + LiveDeploy, + PreviewState, + ServiceRow, +} from "./lib/types"; +import "./styles.css"; + +type Status = { + live: LiveDeploy | null; + services: ServiceRow[]; + remote: string; + productionUrl: string; + errors: string[]; + checkedAt: string; +}; +type JobSummary = Omit; +type HistoryPage = { entries: DeployEntry[]; nextOffset: number | null }; +type Page = "Overview" | "Deploy" | "History" | "Preview"; +const pages: { name: Page; icon: typeof Box }[] = [ + { name: "Overview", icon: LayoutDashboard }, + { name: "Deploy", icon: Rocket }, + { name: "History", icon: History }, + { name: "Preview", icon: Box }, +]; +const running = (job?: JobSummary) => + job?.status === "running" || job?.status === "queued"; +const healthy = (status: string) => + !/unhealthy|exited|dead|restarting/i.test(status) && + /healthy|^up\b/i.test(status); +function relative(iso?: string) { + if (!iso || !Number.isFinite(Date.parse(iso))) return "—"; + const minutes = Math.max( + 0, + Math.floor((Date.now() - Date.parse(iso)) / 60_000), + ); + if (minutes < 1) return "just now"; + if (minutes < 60) return minutes + "m ago"; + if (minutes < 1440) return Math.floor(minutes / 60) + "h ago"; + return Math.floor(minutes / 1440) + "d ago"; +} +function duration(ms?: number) { + if (ms === undefined) return "—"; + const seconds = Math.floor(ms / 1000); + return seconds < 60 + ? seconds + "s" + : Math.floor(seconds / 60) + "m " + (seconds % 60) + "s"; +} +async function request(path: string, init?: RequestInit): Promise { + const response = await fetch(path, { + ...init, + signal: AbortSignal.timeout(35_000), + }); + const data = await response.json(); + if (!response.ok) + throw new Error(data.error ?? "Request failed (" + response.status + ")"); + return data as T; +} +function Badge({ + children, + tone = "", +}: { + children: ReactNode; + tone?: string; +}) { + return ( + + + {children} + + ); +} +function OutLink({ href, children }: { href: string; children: ReactNode }) { + return ( + + {children} + + + ); +} +function Empty({ children }: { children: ReactNode }) { + return
{children}
; +} +function Dialog({ + title, + children, + close, +}: { + title: string; + children: ReactNode; + close: () => void; +}) { + const ref = useRef(null); + useEffect(() => { + ref.current?.showModal(); + }, []); + return ( + +
+

{title}

+ +
+ {children} + + ); +} +function App() { + const initial = window.location.hash.slice(1); + const [page, setPage] = useState( + pages.find((p) => p.name.toLowerCase() === initial)?.name ?? "Overview", + ); + const [status, setStatus] = useState(null); + const [git, setGit] = useState(null); + const [preview, setPreview] = useState(null); + const [history, setHistory] = useState({ + entries: [], + nextOffset: null, + }); + const [historyError, setHistoryError] = useState(""); + const [historyLoading, setHistoryLoading] = useState(false); + const [jobs, setJobs] = useState([]); + const [selectedJob, setSelectedJob] = useState(); + const [streamJob, setStreamJob] = useState(); + const [logs, setLogs] = useState([]); + const [streamError, setStreamError] = useState(""); + const [error, setError] = useState(""); + const [refreshing, setRefreshing] = useState(false); + const refreshingRef = useRef(false); + const [pending, setPending] = useState(false); + const [fast, setFast] = useState(false); + const [skipPreview, setSkipPreview] = useState(false); + const [confirmation, setConfirmation] = useState(null); + const [detail, setDetail] = useState(null); + const [rollbackConfirmed, setRollbackConfirmed] = useState(false); + const [frame, setFrame] = useState(false); + const [tick, setTick] = useState(Date.now()); + const busy = pending || jobs.some(running) || running(streamJob); + const previewReady = + preview?.status === "running" && + preview.health === "healthy" && + !!git && + preview.fingerprint === git.fingerprint; + + const loadHistory = useCallback(async (offset = 0) => { + setHistoryLoading(true); + setHistoryError(""); + try { + const result = await request( + "/api/history?limit=20&offset=" + offset, + ); + setHistory((current) => ({ + ...result, + entries: offset + ? [...current.entries, ...result.entries] + : result.entries, + })); + } catch (e) { + setHistoryError((e as Error).message); + } finally { + setHistoryLoading(false); + } + }, []); + const refresh = useCallback(async () => { + if (refreshingRef.current) return; + refreshingRef.current = true; + setRefreshing(true); + setError(""); + const results = await Promise.allSettled([ + request("/api/status").then(setStatus), + request("/api/git").then(setGit), + request("/api/preview").then(setPreview), + request<{ jobs: JobSummary[] }>("/api/jobs").then((result) => { + setJobs(result.jobs); + setSelectedJob((current) => current ?? result.jobs[0]?.id); + }), + ]); + const failures = results.flatMap((result) => + result.status === "rejected" ? [(result.reason as Error).message] : [], + ); + if (failures.length) setError(failures.join(" · ")); + refreshingRef.current = false; + setRefreshing(false); + }, []); + useEffect(() => { + void refresh(); + void loadHistory(); + const timer = setInterval(() => { + void refresh(); + }, 15_000); + const clock = setInterval(() => setTick(Date.now()), 1_000); + return () => { + clearInterval(timer); + clearInterval(clock); + }; + }, [refresh, loadHistory]); + useEffect(() => { + const keyboard = (event: KeyboardEvent) => { + if ( + event.key.toLowerCase() === "r" && + !event.metaKey && + !event.ctrlKey && + !event.altKey && + !( + event.target instanceof HTMLElement && + (event.target.matches("input, textarea, select") || + event.target.isContentEditable) + ) && + !document.querySelector("dialog[open]") + ) { + event.preventDefault(); + void refresh(); + void loadHistory(); + } + }; + const hash = () => { + const found = pages.find( + (p) => p.name.toLowerCase() === window.location.hash.slice(1), + ); + if (found) setPage(found.name); + }; + window.addEventListener("keydown", keyboard); + window.addEventListener("hashchange", hash); + return () => { + window.removeEventListener("keydown", keyboard); + window.removeEventListener("hashchange", hash); + }; + }, [refresh, loadHistory]); + useEffect(() => { + if (!selectedJob) return; + setLogs([]); + setStreamJob(undefined); + setStreamError(""); + const source = new EventSource("/api/jobs/" + selectedJob + "/events"); + source.addEventListener("log", (event: MessageEvent) => { + setLogs((current) => + [...current, JSON.parse(event.data) as string].slice(-1000), + ); + }); + source.addEventListener("status", (event: MessageEvent) => { + const job = JSON.parse(event.data) as JobSummary; + setStreamJob(job); + setStreamError(""); + if (!running(job)) { + source.close(); + setJobs((current) => + current.map((entry) => (entry.id === job.id ? job : entry)), + ); + void refresh(); + void loadHistory(); + } + }); + source.onerror = () => + setStreamError("Log connection interrupted; reconnecting…"); + return () => source.close(); + }, [selectedJob, refresh, loadHistory]); + + const navigate = (next: Page) => { + setPage(next); + window.location.hash = next.toLowerCase(); + }; + async function action(path: string, body: Record = {}) { + setPending(true); + setError(""); + try { + const { token } = await request<{ token: string }>("/api/session"); + const { job } = await request<{ job: Job }>(path, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Deploy-Desk-Token": token, + }, + body: JSON.stringify(body), + }); + setSelectedJob(job.id); + setJobs((current) => [ + job, + ...current.filter((entry) => entry.id !== job.id), + ]); + setConfirmation(null); + setDetail(null); + await refresh(); + } catch (e) { + setError((e as Error).message); + } finally { + setPending(false); + } + } + const pageDescriptions: Record = { + Overview: "Production, your checkout, and the next release.", + Deploy: "Review your changes. Preview locally. Ship with confidence.", + History: "Every recorded rollout, with a path back.", + Preview: "The production app, running on your machine.", + }; + const live = status?.live; + const previewControls = ( +
+ + + + {preview?.status === "running" && ( + Open preview + )} +
+ ); + const gitReview = ( +
+
+ + + {git?.branch ?? "Reading checkout…"} + + + {git?.dirty + ? "Uncommitted changes" + : git + ? "Clean checkout" + : "Loading"} + +
+

{git?.message ?? "Loading commit…"}

+

+ + {git?.shortSha ?? "—"} + {git?.author} +

+ {git?.dirty && ( +
+ + {git.changedFiles} changed files · view diff summary + + 
{git.diffStat}
+

+ Untracked files count toward dirty state; the diff summary covers + tracked files. +

+
+ )} +
+ ); + return ( +
+ +
+
+ + + NyumatFlix + Deploy Desk + + + + {status?.remote ?? "leetbot"} + +
+
+
+
+

{page}

+

{pageDescriptions[page]}

+
+
+ + {page !== "Deploy" && ( + + )} +
+
+ {error && ( +
+ {error} + +
+ )} + {git?.dirty && ( +
+ + + {git.changedFiles} local changes will ship with this checkout. + The image will be marked dirty. + +
+ )} + {page === "Overview" && ( + <> +
+
+

Production

+ + {status?.errors.length + ? "Status unavailable" + : live + ? "Deployed" + : status + ? "No deployment" + : "Connecting"} + +
+ {status?.errors.map((message) => ( +

+ {message} +

+ ))} + {live ? ( + <> +
+ + + +
+

+ {live.message || + "Deployment without a commit message"} +

+

+ {live.shortSha} + / + {live.author} + / + {relative(live.deployedAt)} +

+
+
+
+
+
Image
+
+ {live.image} +
+
+
+
Source
+
{live.source}
+
+
+
Deployed
+
{new Date(live.deployedAt).toLocaleString()}
+
+
+ + ) : ( + + {status + ? "No live deployment metadata reported. Service status is shown below." + : "Connecting to your production host…"} + + )} +
+ + + {status + ? "Checked " + relative(status.checkedAt) + : "Waiting for status"} + + + Visit production + +
+
+
+
+
+

Services

+ + {status?.services.length ?? "—"} reported + +
+ {status?.services.length ? ( +
+ {status.services.map((service) => ( +
+ +
+ {service.name} + {service.image} +
+ + {service.status} + +
+ ))} +
+ ) : ( + + {refreshing + ? "Reading service health…" + : "No services reported. Check SSH access and refresh."} + + )} +
+
+
+

Next release

+ +
+ {gitReview} +
+ From this working tree + +
+
+
+ + )} + {page === "Deploy" && ( + <> +
+
+
+ 1 +
+

Review checkout

+

+ These local files are the release source. +

+ {gitReview} +
+
+
+ + {previewReady ? : "2"} + +
+
+

Verify a local preview

+ + {previewReady + ? "Ready to review" + : (preview?.status ?? "idle")} + +
+

+ Build the full app from the production Dockerfile, then + start it and check your changes. +

+ {previewControls} + + {preview?.fingerprint && + git && + preview.fingerprint !== git.fingerprint && ( +

+ Checkout changed since the preview build. Rebuild to + use the gate. +

+ )} +
+
+
+ 3 +
+

Ship to {status?.remote ?? "leetbot"}

+

+ Build and push, sync production, then roll through the + existing health checks. +

+ + +
+
+
+ +
+ + )} + {page === "History" && ( +
+
+

Deployment history

+ Newest first +
+ {historyError && ( +
+ {historyError} + +
+ )} + {history.entries.length ? ( +
+ {history.entries.map((entry, index) => ( + + ))} +
+ ) : ( + + {historyLoading + ? "Reading remote deployment history…" + : historyError + ? "History could not be loaded." + : "No deployment history has been recorded yet."} + + )} + {history.nextOffset !== null && ( +
+ + {history.entries.length} deployments loaded + + +
+ )} +
+ )} + {page === "Preview" && ( +
+
+

Local Docker preview

+ + {preview?.status ?? "Checking"} + +
+
+
+ +
+

{preview?.image ?? "nyumatflix-preview:local"}

+

+ Production Dockerfile · linux/amd64 · local image only +

+
+
+ {preview?.error && ( +
{preview.error}
+ )} +
+
+
Address
+
+ {preview?.url ?? "—"} +
+
+
+
Container health
+
{preview?.health ?? "Not running"}
+
+
+
Image ID
+
+ + {preview?.imageId?.slice(0, 24) ?? "Not built"} + +
+
+
+ {previewControls} +

+ Uses .env.prod unless overridden. Preview may + connect to production services with those credentials. For an + isolated environment, set DEPLOY_DESK_PREVIEW_ENV + . Scrape-dependent pages may need{" "} + bun run dev:stack separately; container URLs + should use host.docker.internal for host + services. +

+ {preview?.status === "running" && ( + + )} +
+ {frame && preview?.status === "running" && ( +